PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1902             if ($path =~ m/^iscsi\:\/\// &&
1903                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1904                 $devicetype = 'generic';
1905             }
1906         }
1907
1908         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         } else {
1911             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1912         }
1913
1914         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1915             $device .= ",rotation_rate=1";
1916         }
1917         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1918
1919     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1920         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1921         my $controller = int($drive->{index} / $maxdev);
1922         my $unit = $drive->{index} % $maxdev;
1923         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1924
1925         $device = "ide-$devicetype";
1926         if ($drive->{interface} eq 'ide') {
1927             $device .= ",bus=ide.$controller,unit=$unit";
1928         } else {
1929             $device .= ",bus=ahci$controller.$unit";
1930         }
1931         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1932
1933         if ($devicetype eq 'hd') {
1934             if (my $model = $drive->{model}) {
1935                 $model = URI::Escape::uri_unescape($model);
1936                 $device .= ",model=$model";
1937             }
1938             if ($drive->{ssd}) {
1939                 $device .= ",rotation_rate=1";
1940             }
1941         }
1942         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1943     } elsif ($drive->{interface} eq 'usb') {
1944         die "implement me";
1945         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1946     } else {
1947         die "unsupported interface type";
1948     }
1949
1950     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1951
1952     if (my $serial = $drive->{serial}) {
1953         $serial = URI::Escape::uri_unescape($serial);
1954         $device .= ",serial=$serial";
1955     }
1956
1957
1958     return $device;
1959 }
1960
1961 sub get_initiator_name {
1962     my $initiator;
1963
1964     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1965     while (defined(my $line = <$fh>)) {
1966         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1967         $initiator = $1;
1968         last;
1969     }
1970     $fh->close();
1971
1972     return $initiator;
1973 }
1974
1975 sub print_drive_full {
1976     my ($storecfg, $vmid, $drive) = @_;
1977
1978     my $path;
1979     my $volid = $drive->{file};
1980     my $format;
1981
1982     if (drive_is_cdrom($drive)) {
1983         $path = get_iso_path($storecfg, $vmid, $volid);
1984     } else {
1985         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1986         if ($storeid) {
1987             $path = PVE::Storage::path($storecfg, $volid);
1988             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1989             $format = qemu_img_format($scfg, $volname);
1990         } else {
1991             $path = $volid;
1992             $format = "raw";
1993         }
1994    }
1995
1996     my $opts = '';
1997     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1998     foreach my $o (@qemu_drive_options) {
1999         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2000     }
2001
2002     # snapshot only accepts on|off
2003     if (defined($drive->{snapshot})) {
2004         my $v = $drive->{snapshot} ? 'on' : 'off';
2005         $opts .= ",snapshot=$v";
2006     }
2007
2008     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2009         my ($dir, $qmpname) = @$type;
2010         if (my $v = $drive->{"mbps$dir"}) {
2011             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2012         }
2013         if (my $v = $drive->{"mbps${dir}_max"}) {
2014             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2015         }
2016         if (my $v = $drive->{"bps${dir}_max_length"}) {
2017             $opts .= ",throttling.bps$qmpname-max-length=$v";
2018         }
2019         if (my $v = $drive->{"iops${dir}"}) {
2020             $opts .= ",throttling.iops$qmpname=$v";
2021         }
2022         if (my $v = $drive->{"iops${dir}_max"}) {
2023             $opts .= ",throttling.iops$qmpname-max=$v";
2024         }
2025         if (my $v = $drive->{"iops${dir}_max_length"}) {
2026             $opts .= ",throttling.iops$qmpname-max-length=$v";
2027         }
2028     }
2029
2030     $opts .= ",format=$format" if $format && !$drive->{format};
2031
2032     my $cache_direct = 0;
2033
2034     if (my $cache = $drive->{cache}) {
2035         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2036     } elsif (!drive_is_cdrom($drive)) {
2037         $opts .= ",cache=none";
2038         $cache_direct = 1;
2039     }
2040
2041     # aio native works only with O_DIRECT
2042     if (!$drive->{aio}) {
2043         if($cache_direct) {
2044             $opts .= ",aio=native";
2045         } else {
2046             $opts .= ",aio=threads";
2047         }
2048     }
2049
2050     if (!drive_is_cdrom($drive)) {
2051         my $detectzeroes;
2052         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2053             $detectzeroes = 'off';
2054         } elsif ($drive->{discard}) {
2055             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2056         } else {
2057             # This used to be our default with discard not being specified:
2058             $detectzeroes = 'on';
2059         }
2060         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2061     }
2062
2063     my $pathinfo = $path ? "file=$path," : '';
2064
2065     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2066 }
2067
2068 sub print_netdevice_full {
2069     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2070
2071     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2072
2073     my $device = $net->{model};
2074     if ($net->{model} eq 'virtio') {
2075          $device = 'virtio-net-pci';
2076      };
2077
2078     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2079     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2080     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2081         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2082         my $vectors = $net->{queues} * 2 + 2;
2083         $tmpstr .= ",vectors=$vectors,mq=on";
2084     }
2085     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2086
2087     if ($use_old_bios_files) {
2088         my $romfile;
2089         if ($device eq 'virtio-net-pci') {
2090             $romfile = 'pxe-virtio.rom';
2091         } elsif ($device eq 'e1000') {
2092             $romfile = 'pxe-e1000.rom';
2093         } elsif ($device eq 'ne2k') {
2094             $romfile = 'pxe-ne2k_pci.rom';
2095         } elsif ($device eq 'pcnet') {
2096             $romfile = 'pxe-pcnet.rom';
2097         } elsif ($device eq 'rtl8139') {
2098             $romfile = 'pxe-rtl8139.rom';
2099         }
2100         $tmpstr .= ",romfile=$romfile" if $romfile;
2101     }
2102
2103     return $tmpstr;
2104 }
2105
2106 sub print_netdev_full {
2107     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2108
2109     my $i = '';
2110     if ($netid =~ m/^net(\d+)$/) {
2111         $i = int($1);
2112     }
2113
2114     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2115
2116     my $ifname = "tap${vmid}i$i";
2117
2118     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2119     die "interface name '$ifname' is too long (max 15 character)\n"
2120         if length($ifname) >= 16;
2121
2122     my $vhostparam = '';
2123     if (is_native($arch)) {
2124         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2125     }
2126
2127     my $vmname = $conf->{name} || "vm$vmid";
2128
2129     my $netdev = "";
2130     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2131
2132     if ($net->{bridge}) {
2133         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2134     } else {
2135         $netdev = "type=user,id=$netid,hostname=$vmname";
2136     }
2137
2138     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2139
2140     return $netdev;
2141 }
2142
2143
2144 sub print_cpu_device {
2145     my ($conf, $id) = @_;
2146
2147     my $kvm = $conf->{kvm} // 1;
2148     my $cpu = $kvm ? "kvm64" : "qemu64";
2149     if (my $cputype = $conf->{cpu}) {
2150         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2151             or die "Cannot parse cpu description: $cputype\n";
2152         $cpu = $cpuconf->{cputype};
2153     }
2154
2155     my $cores = $conf->{cores} || 1;
2156
2157     my $current_core = ($id - 1) % $cores;
2158     my $current_socket = int(($id - 1 - $current_core)/$cores);
2159
2160     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2161 }
2162
2163 my $vga_map = {
2164     'cirrus' => 'cirrus-vga',
2165     'std' => 'VGA',
2166     'vmware' => 'vmware-svga',
2167     'virtio' => 'virtio-vga',
2168 };
2169
2170 sub print_vga_device {
2171     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2172
2173     my $type = $vga_map->{$vga->{type}};
2174     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2175         $type = 'virtio-gpu';
2176     }
2177     my $vgamem_mb = $vga->{memory};
2178     if ($qxlnum) {
2179         $type = $id ? 'qxl' : 'qxl-vga';
2180     }
2181     die "no devicetype for $vga->{type}\n" if !$type;
2182
2183     my $memory = "";
2184     if ($vgamem_mb) {
2185         if ($vga->{type} eq 'virtio') {
2186             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2187             $memory = ",max_hostmem=$bytes";
2188         } elsif ($qxlnum) {
2189             # from https://www.spice-space.org/multiple-monitors.html
2190             $memory = ",vgamem_mb=$vga->{memory}";
2191             my $ram = $vgamem_mb * 4;
2192             my $vram = $vgamem_mb * 2;
2193             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2194         } else {
2195             $memory = ",vgamem_mb=$vga->{memory}";
2196         }
2197     } elsif ($qxlnum && $id) {
2198         $memory = ",ram_size=67108864,vram_size=33554432";
2199     }
2200
2201     my $q35 = machine_type_is_q35($conf);
2202     my $vgaid = "vga" . ($id // '');
2203     my $pciaddr;
2204
2205     if ($q35 && $vgaid eq 'vga') {
2206         # the first display uses pcie.0 bus on q35 machines
2207         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2208     } else {
2209         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2210     }
2211
2212     return "$type,id=${vgaid}${memory}${pciaddr}";
2213 }
2214
2215 sub drive_is_cloudinit {
2216     my ($drive) = @_;
2217     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2218 }
2219
2220 sub drive_is_cdrom {
2221     my ($drive, $exclude_cloudinit) = @_;
2222
2223     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2224
2225     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2226
2227 }
2228
2229 sub parse_number_sets {
2230     my ($set) = @_;
2231     my $res = [];
2232     foreach my $part (split(/;/, $set)) {
2233         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2234             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2235             push @$res, [ $1, $2 ];
2236         } else {
2237             die "invalid range: $part\n";
2238         }
2239     }
2240     return $res;
2241 }
2242
2243 sub parse_numa {
2244     my ($data) = @_;
2245
2246     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2247     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2248     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2249     return $res;
2250 }
2251
2252 sub parse_hostpci {
2253     my ($value) = @_;
2254
2255     return undef if !$value;
2256
2257     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2258
2259     my @idlist = split(/;/, $res->{host});
2260     delete $res->{host};
2261     foreach my $id (@idlist) {
2262         if ($id =~ m/\./) { # full id 00:00.1
2263             push @{$res->{pciid}}, {
2264                 id => $id,
2265             };
2266         } else { # partial id 00:00
2267             $res->{pciid} = PVE::SysFSTools::lspci($id);
2268         }
2269     }
2270     return $res;
2271 }
2272
2273 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2274 sub parse_net {
2275     my ($data) = @_;
2276
2277     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2278     if ($@) {
2279         warn $@;
2280         return undef;
2281     }
2282     if (!defined($res->{macaddr})) {
2283         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2284         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2285     }
2286     return $res;
2287 }
2288
2289 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2290 sub parse_ipconfig {
2291     my ($data) = @_;
2292
2293     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2294     if ($@) {
2295         warn $@;
2296         return undef;
2297     }
2298
2299     if ($res->{gw} && !$res->{ip}) {
2300         warn 'gateway specified without specifying an IP address';
2301         return undef;
2302     }
2303     if ($res->{gw6} && !$res->{ip6}) {
2304         warn 'IPv6 gateway specified without specifying an IPv6 address';
2305         return undef;
2306     }
2307     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2308         warn 'gateway specified together with DHCP';
2309         return undef;
2310     }
2311     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2312         # gw6 + auto/dhcp
2313         warn "IPv6 gateway specified together with $res->{ip6} address";
2314         return undef;
2315     }
2316
2317     if (!$res->{ip} && !$res->{ip6}) {
2318         return { ip => 'dhcp', ip6 => 'dhcp' };
2319     }
2320
2321     return $res;
2322 }
2323
2324 sub print_net {
2325     my $net = shift;
2326
2327     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2328 }
2329
2330 sub add_random_macs {
2331     my ($settings) = @_;
2332
2333     foreach my $opt (keys %$settings) {
2334         next if $opt !~ m/^net(\d+)$/;
2335         my $net = parse_net($settings->{$opt});
2336         next if !$net;
2337         $settings->{$opt} = print_net($net);
2338     }
2339 }
2340
2341 sub vm_is_volid_owner {
2342     my ($storecfg, $vmid, $volid) = @_;
2343
2344     if ($volid !~  m|^/|) {
2345         my ($path, $owner);
2346         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2347         if ($owner && ($owner == $vmid)) {
2348             return 1;
2349         }
2350     }
2351
2352     return undef;
2353 }
2354
2355 sub vmconfig_register_unused_drive {
2356     my ($storecfg, $vmid, $conf, $drive) = @_;
2357
2358     if (drive_is_cloudinit($drive)) {
2359         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2360         warn $@ if $@;
2361     } elsif (!drive_is_cdrom($drive)) {
2362         my $volid = $drive->{file};
2363         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2364             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2365         }
2366     }
2367 }
2368
2369 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2370 my $smbios1_fmt = {
2371     uuid => {
2372         type => 'string',
2373         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2374         format_description => 'UUID',
2375         description => "Set SMBIOS1 UUID.",
2376         optional => 1,
2377     },
2378     version => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 version.",
2383         optional => 1,
2384     },
2385     serial => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 serial number.",
2390         optional => 1,
2391     },
2392     manufacturer => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 manufacturer.",
2397         optional => 1,
2398     },
2399     product => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 product ID.",
2404         optional => 1,
2405     },
2406     sku => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 SKU string.",
2411         optional => 1,
2412     },
2413     family => {
2414         type => 'string',
2415         pattern => '[A-Za-z0-9+\/]+={0,2}',
2416         format_description => 'Base64 encoded string',
2417         description => "Set SMBIOS1 family string.",
2418         optional => 1,
2419     },
2420     base64 => {
2421         type => 'boolean',
2422         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2423         optional => 1,
2424     },
2425 };
2426
2427 sub parse_smbios1 {
2428     my ($data) = @_;
2429
2430     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2431     warn $@ if $@;
2432     return $res;
2433 }
2434
2435 sub print_smbios1 {
2436     my ($smbios1) = @_;
2437     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2438 }
2439
2440 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2441
2442 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2443 sub verify_bootdisk {
2444     my ($value, $noerr) = @_;
2445
2446     return $value if is_valid_drivename($value);
2447
2448     return undef if $noerr;
2449
2450     die "invalid boot disk '$value'\n";
2451 }
2452
2453 sub parse_watchdog {
2454     my ($value) = @_;
2455
2456     return undef if !$value;
2457
2458     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2459     warn $@ if $@;
2460     return $res;
2461 }
2462
2463 sub parse_guest_agent {
2464     my ($value) = @_;
2465
2466     return {} if !defined($value->{agent});
2467
2468     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2469     warn $@ if $@;
2470
2471     # if the agent is disabled ignore the other potentially set properties
2472     return {} if !$res->{enabled};
2473     return $res;
2474 }
2475
2476 sub parse_vga {
2477     my ($value) = @_;
2478
2479     return {} if !$value;
2480     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2481     warn $@ if $@;
2482     return $res;
2483 }
2484
2485 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2486 sub verify_usb_device {
2487     my ($value, $noerr) = @_;
2488
2489     return $value if parse_usb_device($value);
2490
2491     return undef if $noerr;
2492
2493     die "unable to parse usb device\n";
2494 }
2495
2496 # add JSON properties for create and set function
2497 sub json_config_properties {
2498     my $prop = shift;
2499
2500     foreach my $opt (keys %$confdesc) {
2501         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2502         $prop->{$opt} = $confdesc->{$opt};
2503     }
2504
2505     return $prop;
2506 }
2507
2508 # return copy of $confdesc_cloudinit to generate documentation
2509 sub cloudinit_config_properties {
2510
2511     return dclone($confdesc_cloudinit);
2512 }
2513
2514 sub check_type {
2515     my ($key, $value) = @_;
2516
2517     die "unknown setting '$key'\n" if !$confdesc->{$key};
2518
2519     my $type = $confdesc->{$key}->{type};
2520
2521     if (!defined($value)) {
2522         die "got undefined value\n";
2523     }
2524
2525     if ($value =~ m/[\n\r]/) {
2526         die "property contains a line feed\n";
2527     }
2528
2529     if ($type eq 'boolean') {
2530         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2531         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2532         die "type check ('boolean') failed - got '$value'\n";
2533     } elsif ($type eq 'integer') {
2534         return int($1) if $value =~ m/^(\d+)$/;
2535         die "type check ('integer') failed - got '$value'\n";
2536     } elsif ($type eq 'number') {
2537         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2538         die "type check ('number') failed - got '$value'\n";
2539     } elsif ($type eq 'string') {
2540         if (my $fmt = $confdesc->{$key}->{format}) {
2541             PVE::JSONSchema::check_format($fmt, $value);
2542             return $value;
2543         }
2544         $value =~ s/^\"(.*)\"$/$1/;
2545         return $value;
2546     } else {
2547         die "internal error"
2548     }
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conf = PVE::QemuConfig->load_config($vmid);
2555
2556     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2557
2558     if ($conf->{template}) {
2559         # check if any base image is still used by a linked clone
2560         foreach_drive($conf, sub {
2561                 my ($ds, $drive) = @_;
2562
2563                 return if drive_is_cdrom($drive);
2564
2565                 my $volid = $drive->{file};
2566
2567                 return if !$volid || $volid =~ m|^/|;
2568
2569                 die "base volume '$volid' is still in use by linked cloned\n"
2570                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2571
2572         });
2573     }
2574
2575     # only remove disks owned by this VM
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578
2579         return if drive_is_cdrom($drive, 1);
2580
2581         my $volid = $drive->{file};
2582
2583         return if !$volid || $volid =~ m|^/|;
2584
2585         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2586         return if !$path || !$owner || ($owner != $vmid);
2587
2588         eval {
2589             PVE::Storage::vdisk_free($storecfg, $volid);
2590         };
2591         warn "Could not remove disk '$volid', check manually: $@" if $@;
2592
2593     });
2594
2595     if ($keep_empty_config) {
2596         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2597     } else {
2598         PVE::QemuConfig->destroy_config($vmid);
2599     }
2600
2601     # also remove unused disk
2602     eval {
2603         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2604
2605         eval {
2606             PVE::Storage::foreach_volid($dl, sub {
2607                 my ($volid, $sid, $volname, $d) = @_;
2608                 PVE::Storage::vdisk_free($storecfg, $volid);
2609             });
2610         };
2611         warn $@ if $@;
2612
2613     };
2614     warn $@ if $@;
2615 }
2616
2617 sub parse_vm_config {
2618     my ($filename, $raw) = @_;
2619
2620     return undef if !defined($raw);
2621
2622     my $res = {
2623         digest => Digest::SHA::sha1_hex($raw),
2624         snapshots => {},
2625         pending => {},
2626     };
2627
2628     $filename =~ m|/qemu-server/(\d+)\.conf$|
2629         || die "got strange filename '$filename'";
2630
2631     my $vmid = $1;
2632
2633     my $conf = $res;
2634     my $descr;
2635     my $section = '';
2636
2637     my @lines = split(/\n/, $raw);
2638     foreach my $line (@lines) {
2639         next if $line =~ m/^\s*$/;
2640
2641         if ($line =~ m/^\[PENDING\]\s*$/i) {
2642             $section = 'pending';
2643             if (defined($descr)) {
2644                 $descr =~ s/\s+$//;
2645                 $conf->{description} = $descr;
2646             }
2647             $descr = undef;
2648             $conf = $res->{$section} = {};
2649             next;
2650
2651         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2652             $section = $1;
2653             if (defined($descr)) {
2654                 $descr =~ s/\s+$//;
2655                 $conf->{description} = $descr;
2656             }
2657             $descr = undef;
2658             $conf = $res->{snapshots}->{$section} = {};
2659             next;
2660         }
2661
2662         if ($line =~ m/^\#(.*)\s*$/) {
2663             $descr = '' if !defined($descr);
2664             $descr .= PVE::Tools::decode_text($1) . "\n";
2665             next;
2666         }
2667
2668         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2669             $descr = '' if !defined($descr);
2670             $descr .= PVE::Tools::decode_text($2);
2671         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2672             $conf->{snapstate} = $1;
2673         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2674             my $key = $1;
2675             my $value = $2;
2676             $conf->{$key} = $value;
2677         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2678             my $value = $1;
2679             if ($section eq 'pending') {
2680                 $conf->{delete} = $value; # we parse this later
2681             } else {
2682                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2683             }
2684         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2685             my $key = $1;
2686             my $value = $2;
2687             eval { $value = check_type($key, $value); };
2688             if ($@) {
2689                 warn "vm $vmid - unable to parse value of '$key' - $@";
2690             } else {
2691                 $key = 'ide2' if $key eq 'cdrom';
2692                 my $fmt = $confdesc->{$key}->{format};
2693                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2694                     my $v = parse_drive($key, $value);
2695                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2696                         $v->{file} = $volid;
2697                         $value = print_drive($vmid, $v);
2698                     } else {
2699                         warn "vm $vmid - unable to parse value of '$key'\n";
2700                         next;
2701                     }
2702                 }
2703
2704                 $conf->{$key} = $value;
2705             }
2706         }
2707     }
2708
2709     if (defined($descr)) {
2710         $descr =~ s/\s+$//;
2711         $conf->{description} = $descr;
2712     }
2713     delete $res->{snapstate}; # just to be sure
2714
2715     return $res;
2716 }
2717
2718 sub write_vm_config {
2719     my ($filename, $conf) = @_;
2720
2721     delete $conf->{snapstate}; # just to be sure
2722
2723     if ($conf->{cdrom}) {
2724         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2725         $conf->{ide2} = $conf->{cdrom};
2726         delete $conf->{cdrom};
2727     }
2728
2729     # we do not use 'smp' any longer
2730     if ($conf->{sockets}) {
2731         delete $conf->{smp};
2732     } elsif ($conf->{smp}) {
2733         $conf->{sockets} = $conf->{smp};
2734         delete $conf->{cores};
2735         delete $conf->{smp};
2736     }
2737
2738     my $used_volids = {};
2739
2740     my $cleanup_config = sub {
2741         my ($cref, $pending, $snapname) = @_;
2742
2743         foreach my $key (keys %$cref) {
2744             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2745                 $key eq 'snapstate' || $key eq 'pending';
2746             my $value = $cref->{$key};
2747             if ($key eq 'delete') {
2748                 die "propertry 'delete' is only allowed in [PENDING]\n"
2749                     if !$pending;
2750                 # fixme: check syntax?
2751                 next;
2752             }
2753             eval { $value = check_type($key, $value); };
2754             die "unable to parse value of '$key' - $@" if $@;
2755
2756             $cref->{$key} = $value;
2757
2758             if (!$snapname && is_valid_drivename($key)) {
2759                 my $drive = parse_drive($key, $value);
2760                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2761             }
2762         }
2763     };
2764
2765     &$cleanup_config($conf);
2766
2767     &$cleanup_config($conf->{pending}, 1);
2768
2769     foreach my $snapname (keys %{$conf->{snapshots}}) {
2770         die "internal error" if $snapname eq 'pending';
2771         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2772     }
2773
2774     # remove 'unusedX' settings if we re-add a volume
2775     foreach my $key (keys %$conf) {
2776         my $value = $conf->{$key};
2777         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2778             delete $conf->{$key};
2779         }
2780     }
2781
2782     my $generate_raw_config = sub {
2783         my ($conf, $pending) = @_;
2784
2785         my $raw = '';
2786
2787         # add description as comment to top of file
2788         if (defined(my $descr = $conf->{description})) {
2789             if ($descr) {
2790                 foreach my $cl (split(/\n/, $descr)) {
2791                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2792                 }
2793             } else {
2794                 $raw .= "#\n" if $pending;
2795             }
2796         }
2797
2798         foreach my $key (sort keys %$conf) {
2799             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2800             $raw .= "$key: $conf->{$key}\n";
2801         }
2802         return $raw;
2803     };
2804
2805     my $raw = &$generate_raw_config($conf);
2806
2807     if (scalar(keys %{$conf->{pending}})){
2808         $raw .= "\n[PENDING]\n";
2809         $raw .= &$generate_raw_config($conf->{pending}, 1);
2810     }
2811
2812     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2813         $raw .= "\n[$snapname]\n";
2814         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2815     }
2816
2817     return $raw;
2818 }
2819
2820 sub load_defaults {
2821
2822     my $res = {};
2823
2824     # we use static defaults from our JSON schema configuration
2825     foreach my $key (keys %$confdesc) {
2826         if (defined(my $default = $confdesc->{$key}->{default})) {
2827             $res->{$key} = $default;
2828         }
2829     }
2830
2831     return $res;
2832 }
2833
2834 sub config_list {
2835     my $vmlist = PVE::Cluster::get_vmlist();
2836     my $res = {};
2837     return $res if !$vmlist || !$vmlist->{ids};
2838     my $ids = $vmlist->{ids};
2839
2840     foreach my $vmid (keys %$ids) {
2841         my $d = $ids->{$vmid};
2842         next if !$d->{node} || $d->{node} ne $nodename;
2843         next if !$d->{type} || $d->{type} ne 'qemu';
2844         $res->{$vmid}->{exists} = 1;
2845     }
2846     return $res;
2847 }
2848
2849 # test if VM uses local resources (to prevent migration)
2850 sub check_local_resources {
2851     my ($conf, $noerr) = @_;
2852
2853     my @loc_res = ();
2854
2855     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2856     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2857
2858     push @loc_res, "ivshmem" if $conf->{ivshmem};
2859
2860     foreach my $k (keys %$conf) {
2861         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2862         # sockets are safe: they will recreated be on the target side post-migrate
2863         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2864         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2865     }
2866
2867     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2868
2869     return \@loc_res;
2870 }
2871
2872 # check if used storages are available on all nodes (use by migrate)
2873 sub check_storage_availability {
2874     my ($storecfg, $conf, $node) = @_;
2875
2876     foreach_drive($conf, sub {
2877         my ($ds, $drive) = @_;
2878
2879         my $volid = $drive->{file};
2880         return if !$volid;
2881
2882         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2883         return if !$sid;
2884
2885         # check if storage is available on both nodes
2886         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2887         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2888    });
2889 }
2890
2891 # list nodes where all VM images are available (used by has_feature API)
2892 sub shared_nodes {
2893     my ($conf, $storecfg) = @_;
2894
2895     my $nodelist = PVE::Cluster::get_nodelist();
2896     my $nodehash = { map { $_ => 1 } @$nodelist };
2897     my $nodename = PVE::INotify::nodename();
2898
2899     foreach_drive($conf, sub {
2900         my ($ds, $drive) = @_;
2901
2902         my $volid = $drive->{file};
2903         return if !$volid;
2904
2905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906         if ($storeid) {
2907             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2908             if ($scfg->{disable}) {
2909                 $nodehash = {};
2910             } elsif (my $avail = $scfg->{nodes}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     delete $nodehash->{$node} if !$avail->{$node};
2913                 }
2914             } elsif (!$scfg->{shared}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     delete $nodehash->{$node} if $node ne $nodename
2917                 }
2918             }
2919         }
2920     });
2921
2922     return $nodehash
2923 }
2924
2925 sub check_local_storage_availability {
2926     my ($conf, $storecfg) = @_;
2927
2928     my $nodelist = PVE::Cluster::get_nodelist();
2929     my $nodehash = { map { $_ => {} } @$nodelist };
2930
2931     foreach_drive($conf, sub {
2932         my ($ds, $drive) = @_;
2933
2934         my $volid = $drive->{file};
2935         return if !$volid;
2936
2937         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2938         if ($storeid) {
2939             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2940
2941             if ($scfg->{disable}) {
2942                 foreach my $node (keys %$nodehash) {
2943                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2944                 }
2945             } elsif (my $avail = $scfg->{nodes}) {
2946                 foreach my $node (keys %$nodehash) {
2947                     if (!$avail->{$node}) {
2948                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2949                     }
2950                 }
2951             }
2952         }
2953     });
2954
2955     foreach my $node (values %$nodehash) {
2956         if (my $unavail = $node->{unavailable_storages}) {
2957             $node->{unavailable_storages} = [ sort keys %$unavail ];
2958         }
2959     }
2960
2961     return $nodehash
2962 }
2963
2964 sub check_cmdline {
2965     my ($pidfile, $pid) = @_;
2966
2967     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2968     if (defined($fh)) {
2969         my $line = <$fh>;
2970         $fh->close;
2971         return undef if !$line;
2972         my @param = split(/\0/, $line);
2973
2974         my $cmd = $param[0];
2975         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2976
2977         for (my $i = 0; $i < scalar (@param); $i++) {
2978             my $p = $param[$i];
2979             next if !$p;
2980             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2981                 my $p = $param[$i+1];
2982                 return 1 if $p && ($p eq $pidfile);
2983                 return undef;
2984             }
2985         }
2986     }
2987     return undef;
2988 }
2989
2990 sub check_running {
2991     my ($vmid, $nocheck, $node) = @_;
2992
2993     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2994
2995     die "unable to find configuration file for VM $vmid - no such machine\n"
2996         if !$nocheck && ! -f $filename;
2997
2998     my $pidfile = pidfile_name($vmid);
2999
3000     if (my $fd = IO::File->new("<$pidfile")) {
3001         my $st = stat($fd);
3002         my $line = <$fd>;
3003         close($fd);
3004
3005         my $mtime = $st->mtime;
3006         if ($mtime > time()) {
3007             warn "file '$filename' modified in future\n";
3008         }
3009
3010         if ($line =~ m/^(\d+)$/) {
3011             my $pid = $1;
3012             if (check_cmdline($pidfile, $pid)) {
3013                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3014                     return $pid;
3015                 }
3016             }
3017         }
3018     }
3019
3020     return undef;
3021 }
3022
3023 sub vzlist {
3024
3025     my $vzlist = config_list();
3026
3027     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3028
3029     while (defined(my $de = $fd->read)) {
3030         next if $de !~ m/^(\d+)\.pid$/;
3031         my $vmid = $1;
3032         next if !defined($vzlist->{$vmid});
3033         if (my $pid = check_running($vmid)) {
3034             $vzlist->{$vmid}->{pid} = $pid;
3035         }
3036     }
3037
3038     return $vzlist;
3039 }
3040
3041 sub disksize {
3042     my ($storecfg, $conf) = @_;
3043
3044     my $bootdisk = $conf->{bootdisk};
3045     return undef if !$bootdisk;
3046     return undef if !is_valid_drivename($bootdisk);
3047
3048     return undef if !$conf->{$bootdisk};
3049
3050     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3051     return undef if !defined($drive);
3052
3053     return undef if drive_is_cdrom($drive);
3054
3055     my $volid = $drive->{file};
3056     return undef if !$volid;
3057
3058     return $drive->{size};
3059 }
3060
3061 our $vmstatus_return_properties = {
3062     vmid => get_standard_option('pve-vmid'),
3063     status => {
3064         description => "Qemu process status.",
3065         type => 'string',
3066         enum => ['stopped', 'running'],
3067     },
3068     maxmem => {
3069         description => "Maximum memory in bytes.",
3070         type => 'integer',
3071         optional => 1,
3072         renderer => 'bytes',
3073     },
3074     maxdisk => {
3075         description => "Root disk size in bytes.",
3076         type => 'integer',
3077         optional => 1,
3078         renderer => 'bytes',
3079     },
3080     name => {
3081         description => "VM name.",
3082         type => 'string',
3083         optional => 1,
3084     },
3085     qmpstatus => {
3086         description => "Qemu QMP agent status.",
3087         type => 'string',
3088         optional => 1,
3089     },
3090     pid => {
3091         description => "PID of running qemu process.",
3092         type => 'integer',
3093         optional => 1,
3094     },
3095     uptime => {
3096         description => "Uptime.",
3097         type => 'integer',
3098         optional => 1,
3099         renderer => 'duration',
3100     },
3101     cpus => {
3102         description => "Maximum usable CPUs.",
3103         type => 'number',
3104         optional => 1,
3105     },
3106     lock => {
3107         description => "The current config lock, if any.",
3108         type => 'string',
3109         optional => 1,
3110     }
3111 };
3112
3113 my $last_proc_pid_stat;
3114
3115 # get VM status information
3116 # This must be fast and should not block ($full == false)
3117 # We only query KVM using QMP if $full == true (this can be slow)
3118 sub vmstatus {
3119     my ($opt_vmid, $full) = @_;
3120
3121     my $res = {};
3122
3123     my $storecfg = PVE::Storage::config();
3124
3125     my $list = vzlist();
3126     my $defaults = load_defaults();
3127
3128     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3129
3130     my $cpucount = $cpuinfo->{cpus} || 1;
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134
3135         my $conf = PVE::QemuConfig->load_config($vmid);
3136
3137         my $d = { vmid => $vmid };
3138         $d->{pid} = $list->{$vmid}->{pid};
3139
3140         # fixme: better status?
3141         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3142
3143         my $size = disksize($storecfg, $conf);
3144         if (defined($size)) {
3145             $d->{disk} = 0; # no info available
3146             $d->{maxdisk} = $size;
3147         } else {
3148             $d->{disk} = 0;
3149             $d->{maxdisk} = 0;
3150         }
3151
3152         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3153             * ($conf->{cores} || $defaults->{cores});
3154         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3155         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3156
3157         $d->{name} = $conf->{name} || "VM $vmid";
3158         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3159             : $defaults->{memory}*(1024*1024);
3160
3161         if ($conf->{balloon}) {
3162             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3163             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3164                 : $defaults->{shares};
3165         }
3166
3167         $d->{uptime} = 0;
3168         $d->{cpu} = 0;
3169         $d->{mem} = 0;
3170
3171         $d->{netout} = 0;
3172         $d->{netin} = 0;
3173
3174         $d->{diskread} = 0;
3175         $d->{diskwrite} = 0;
3176
3177         $d->{template} = PVE::QemuConfig->is_template($conf);
3178
3179         $d->{serial} = 1 if conf_has_serial($conf);
3180         $d->{lock} = $conf->{lock} if $conf->{lock};
3181
3182         $res->{$vmid} = $d;
3183     }
3184
3185     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3186     foreach my $dev (keys %$netdev) {
3187         next if $dev !~ m/^tap([1-9]\d*)i/;
3188         my $vmid = $1;
3189         my $d = $res->{$vmid};
3190         next if !$d;
3191
3192         $d->{netout} += $netdev->{$dev}->{receive};
3193         $d->{netin} += $netdev->{$dev}->{transmit};
3194
3195         if ($full) {
3196             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3197             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3198         }
3199
3200     }
3201
3202     my $ctime = gettimeofday;
3203
3204     foreach my $vmid (keys %$list) {
3205
3206         my $d = $res->{$vmid};
3207         my $pid = $d->{pid};
3208         next if !$pid;
3209
3210         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3211         next if !$pstat; # not running
3212
3213         my $used = $pstat->{utime} + $pstat->{stime};
3214
3215         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3216
3217         if ($pstat->{vsize}) {
3218             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3219         }
3220
3221         my $old = $last_proc_pid_stat->{$pid};
3222         if (!$old) {
3223             $last_proc_pid_stat->{$pid} = {
3224                 time => $ctime,
3225                 used => $used,
3226                 cpu => 0,
3227             };
3228             next;
3229         }
3230
3231         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3232
3233         if ($dtime > 1000) {
3234             my $dutime = $used -  $old->{used};
3235
3236             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3237             $last_proc_pid_stat->{$pid} = {
3238                 time => $ctime,
3239                 used => $used,
3240                 cpu => $d->{cpu},
3241             };
3242         } else {
3243             $d->{cpu} = $old->{cpu};
3244         }
3245     }
3246
3247     return $res if !$full;
3248
3249     my $qmpclient = PVE::QMPClient->new();
3250
3251     my $ballooncb = sub {
3252         my ($vmid, $resp) = @_;
3253
3254         my $info = $resp->{'return'};
3255         return if !$info->{max_mem};
3256
3257         my $d = $res->{$vmid};
3258
3259         # use memory assigned to VM
3260         $d->{maxmem} = $info->{max_mem};
3261         $d->{balloon} = $info->{actual};
3262
3263         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3264             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3265             $d->{freemem} = $info->{free_mem};
3266         }
3267
3268         $d->{ballooninfo} = $info;
3269     };
3270
3271     my $blockstatscb = sub {
3272         my ($vmid, $resp) = @_;
3273         my $data = $resp->{'return'} || [];
3274         my $totalrdbytes = 0;
3275         my $totalwrbytes = 0;
3276
3277         for my $blockstat (@$data) {
3278             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3279             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3280
3281             $blockstat->{device} =~ s/drive-//;
3282             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3283         }
3284         $res->{$vmid}->{diskread} = $totalrdbytes;
3285         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3286     };
3287
3288     my $statuscb = sub {
3289         my ($vmid, $resp) = @_;
3290
3291         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3292         # this fails if ballon driver is not loaded, so this must be
3293         # the last commnand (following command are aborted if this fails).
3294         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3295
3296         my $status = 'unknown';
3297         if (!defined($status = $resp->{'return'}->{status})) {
3298             warn "unable to get VM status\n";
3299             return;
3300         }
3301
3302         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3303     };
3304
3305     foreach my $vmid (keys %$list) {
3306         next if $opt_vmid && ($vmid ne $opt_vmid);
3307         next if !$res->{$vmid}->{pid}; # not running
3308         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3309     }
3310
3311     $qmpclient->queue_execute(undef, 2);
3312
3313     foreach my $vmid (keys %$list) {
3314         next if $opt_vmid && ($vmid ne $opt_vmid);
3315         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3316     }
3317
3318     return $res;
3319 }
3320
3321 sub foreach_drive {
3322     my ($conf, $func, @param) = @_;
3323
3324     foreach my $ds (valid_drive_names()) {
3325         next if !defined($conf->{$ds});
3326
3327         my $drive = parse_drive($ds, $conf->{$ds});
3328         next if !$drive;
3329
3330         &$func($ds, $drive, @param);
3331     }
3332 }
3333
3334 sub foreach_volid {
3335     my ($conf, $func, @param) = @_;
3336
3337     my $volhash = {};
3338
3339     my $test_volid = sub {
3340         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3341
3342         return if !$volid;
3343
3344         $volhash->{$volid}->{cdrom} //= 1;
3345         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3346
3347         $volhash->{$volid}->{replicate} //= 0;
3348         $volhash->{$volid}->{replicate} = 1 if $replicate;
3349
3350         $volhash->{$volid}->{shared} //= 0;
3351         $volhash->{$volid}->{shared} = 1 if $shared;
3352
3353         $volhash->{$volid}->{referenced_in_config} //= 0;
3354         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3355
3356         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3357             if defined($snapname);
3358         $volhash->{$volid}->{size} = $size if $size;
3359     };
3360
3361     foreach_drive($conf, sub {
3362         my ($ds, $drive) = @_;
3363         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3364     });
3365
3366     foreach my $snapname (keys %{$conf->{snapshots}}) {
3367         my $snap = $conf->{snapshots}->{$snapname};
3368         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3369         foreach_drive($snap, sub {
3370             my ($ds, $drive) = @_;
3371             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3372         });
3373     }
3374
3375     foreach my $volid (keys %$volhash) {
3376         &$func($volid, $volhash->{$volid}, @param);
3377     }
3378 }
3379
3380 sub conf_has_serial {
3381     my ($conf) = @_;
3382
3383     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3384         if ($conf->{"serial$i"}) {
3385             return 1;
3386         }
3387     }
3388
3389     return 0;
3390 }
3391
3392 sub conf_has_audio {
3393     my ($conf, $id) = @_;
3394
3395     $id //= 0;
3396     my $audio = $conf->{"audio$id"};
3397     return undef if !defined($audio);
3398
3399     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3400     my $audiodriver = $audioproperties->{driver} // 'spice';
3401
3402     return {
3403         dev => $audioproperties->{device},
3404         dev_id => "audiodev$id",
3405         backend => $audiodriver,
3406         backend_id => "$audiodriver-backend${id}",
3407     };
3408 }
3409
3410 sub vga_conf_has_spice {
3411     my ($vga) = @_;
3412
3413     my $vgaconf = parse_vga($vga);
3414     my $vgatype = $vgaconf->{type};
3415     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3416
3417     return $1 || 1;
3418 }
3419
3420 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3421 sub get_host_arch() {
3422     $host_arch = (POSIX::uname())[4] if !$host_arch;
3423     return $host_arch;
3424 }
3425
3426 sub is_native($) {
3427     my ($arch) = @_;
3428     return get_host_arch() eq $arch;
3429 }
3430
3431 my $default_machines = {
3432     x86_64 => 'pc',
3433     aarch64 => 'virt',
3434 };
3435
3436 sub get_basic_machine_info {
3437     my ($conf, $forcemachine) = @_;
3438
3439     my $arch = $conf->{arch} // get_host_arch();
3440     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3441     return ($arch, $machine);
3442 }
3443
3444 sub get_ovmf_files($) {
3445     my ($arch) = @_;
3446
3447     my $ovmf = $OVMF->{$arch}
3448         or die "no OVMF images known for architecture '$arch'\n";
3449
3450     return @$ovmf;
3451 }
3452
3453 my $Arch2Qemu = {
3454     aarch64 => '/usr/bin/qemu-system-aarch64',
3455     x86_64 => '/usr/bin/qemu-system-x86_64',
3456 };
3457 sub get_command_for_arch($) {
3458     my ($arch) = @_;
3459     return '/usr/bin/kvm' if is_native($arch);
3460
3461     my $cmd = $Arch2Qemu->{$arch}
3462         or die "don't know how to emulate architecture '$arch'\n";
3463     return $cmd;
3464 }
3465
3466 sub get_cpu_options {
3467     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3468
3469     my $cpuFlags = [];
3470     my $ostype = $conf->{ostype};
3471
3472     my $cpu = $kvm ? "kvm64" : "qemu64";
3473     if ($arch eq 'aarch64') {
3474         $cpu = 'cortex-a57';
3475     }
3476     my $hv_vendor_id;
3477     if (my $cputype = $conf->{cpu}) {
3478         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3479             or die "Cannot parse cpu description: $cputype\n";
3480         $cpu = $cpuconf->{cputype};
3481         $kvm_off = 1 if $cpuconf->{hidden};
3482         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3483
3484         if (defined(my $flags = $cpuconf->{flags})) {
3485             push @$cpuFlags, split(";", $flags);
3486         }
3487     }
3488
3489     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3490
3491     push @$cpuFlags , '-x2apic'
3492         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3493
3494     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3495
3496     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3497
3498     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3499
3500         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3501         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3502     }
3503
3504     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3505
3506     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3507
3508     push @$cpuFlags, 'kvm=off' if $kvm_off;
3509
3510     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3511         push @$cpuFlags, "vendor=${cpu_vendor}"
3512             if $cpu_vendor ne 'default';
3513     } elsif ($arch ne 'aarch64') {
3514         die "internal error"; # should not happen
3515     }
3516
3517     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3518
3519     return ('-cpu', $cpu);
3520 }
3521
3522 sub config_to_command {
3523     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3524
3525     my $cmd = [];
3526     my $globalFlags = [];
3527     my $machineFlags = [];
3528     my $rtcFlags = [];
3529     my $devices = [];
3530     my $pciaddr = '';
3531     my $bridges = {};
3532     my $vernum = 0; # unknown
3533     my $ostype = $conf->{ostype};
3534     my $winversion = windows_version($ostype);
3535     my $kvm = $conf->{kvm};
3536
3537     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3538     my $kvm_binary = get_command_for_arch($arch);
3539     my $kvmver = kvm_user_version($kvm_binary);
3540     $kvm //= 1 if is_native($arch);
3541
3542     if ($kvm) {
3543         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3544             if !defined kvm_version();
3545     }
3546
3547     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3548         $vernum = $1*1000000+$2*1000;
3549     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3550         $vernum = $1*1000000+$2*1000+$3;
3551     }
3552
3553     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3554
3555     my $q35 = machine_type_is_q35($conf);
3556     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3557     my $use_old_bios_files = undef;
3558     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3559
3560     my $cpuunits = defined($conf->{cpuunits}) ?
3561             $conf->{cpuunits} : $defaults->{cpuunits};
3562
3563     push @$cmd, $kvm_binary;
3564
3565     push @$cmd, '-id', $vmid;
3566
3567     my $vmname = $conf->{name} || "vm$vmid";
3568
3569     push @$cmd, '-name', $vmname;
3570
3571     my $use_virtio = 0;
3572
3573     my $qmpsocket = qmp_socket($vmid);
3574     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3575     push @$cmd, '-mon', "chardev=qmp,mode=control";
3576
3577     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3578         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3579         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3580     }
3581
3582     push @$cmd, '-pidfile' , pidfile_name($vmid);
3583
3584     push @$cmd, '-daemonize';
3585
3586     if ($conf->{smbios1}) {
3587         my $smbios_conf = parse_smbios1($conf->{smbios1});
3588         if ($smbios_conf->{base64}) {
3589             # Do not pass base64 flag to qemu
3590             delete $smbios_conf->{base64};
3591             my $smbios_string = "";
3592             foreach my $key (keys %$smbios_conf) {
3593                 my $value;
3594                 if ($key eq "uuid") {
3595                     $value = $smbios_conf->{uuid}
3596                 } else {
3597                     $value = decode_base64($smbios_conf->{$key});
3598                 }
3599                 # qemu accepts any binary data, only commas need escaping by double comma
3600                 $value =~ s/,/,,/g;
3601                 $smbios_string .= "," . $key . "=" . $value if $value;
3602             }
3603             push @$cmd, '-smbios', "type=1" . $smbios_string;
3604         } else {
3605             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3606         }
3607     }
3608
3609     if ($conf->{vmgenid}) {
3610         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3611     }
3612
3613     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3614     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3615         die "uefi base image not found\n" if ! -f $ovmf_code;
3616
3617         my $path;
3618         my $format;
3619         if (my $efidisk = $conf->{efidisk0}) {
3620             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3621             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3622             $format = $d->{format};
3623             if ($storeid) {
3624                 $path = PVE::Storage::path($storecfg, $d->{file});
3625                 if (!defined($format)) {
3626                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3627                     $format = qemu_img_format($scfg, $volname);
3628                 }
3629             } else {
3630                 $path = $d->{file};
3631                 die "efidisk format must be specified\n"
3632                     if !defined($format);
3633             }
3634         } else {
3635             warn "no efidisk configured! Using temporary efivars disk.\n";
3636             $path = "/tmp/$vmid-ovmf.fd";
3637             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3638             $format = 'raw';
3639         }
3640
3641         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3642         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3643     }
3644
3645     # load q35 config
3646     if ($q35) {
3647         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3648         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3649             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3650         } else {
3651             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3652         }
3653     }
3654
3655     # add usb controllers
3656     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3657     push @$devices, @usbcontrollers if @usbcontrollers;
3658     my $vga = parse_vga($conf->{vga});
3659
3660     my $qxlnum = vga_conf_has_spice($conf->{vga});
3661     $vga->{type} = 'qxl' if $qxlnum;
3662
3663     if (!$vga->{type}) {
3664         if ($arch eq 'aarch64') {
3665             $vga->{type} = 'virtio';
3666         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3667             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3668         } else {
3669             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3670         }
3671     }
3672
3673     # enable absolute mouse coordinates (needed by vnc)
3674     my $tablet;
3675     if (defined($conf->{tablet})) {
3676         $tablet = $conf->{tablet};
3677     } else {
3678         $tablet = $defaults->{tablet};
3679         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3680         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3681     }
3682
3683     if ($tablet) {
3684         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3685         my $kbd = print_keyboarddevice_full($conf, $arch);
3686         push @$devices, '-device', $kbd if defined($kbd);
3687     }
3688
3689     my $kvm_off = 0;
3690     my $gpu_passthrough;
3691
3692     # host pci devices
3693     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3694         my $id = "hostpci$i";
3695         my $d = parse_hostpci($conf->{$id});
3696         next if !$d;
3697
3698         if (my $pcie = $d->{pcie}) {
3699             die "q35 machine model is not enabled" if !$q35;
3700             # win7 wants to have the pcie devices directly on the pcie bus
3701             # instead of in the root port
3702             if ($winversion == 7) {
3703                 $pciaddr = print_pcie_addr("${id}bus0");
3704             } else {
3705                 # add more root ports if needed, 4 are present by default
3706                 # by pve-q35 cfgs, rest added here on demand.
3707                 if ($i > 3) {
3708                     push @$devices, '-device', print_pcie_root_port($i);
3709                 }
3710                 $pciaddr = print_pcie_addr($id);
3711             }
3712         } else {
3713             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3714         }
3715
3716         my $xvga = '';
3717         if ($d->{'x-vga'}) {
3718             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3719             $kvm_off = 1;
3720             $vga->{type} = 'none' if !defined($conf->{vga});
3721             $gpu_passthrough = 1;
3722         }
3723
3724         my $pcidevices = $d->{pciid};
3725         my $multifunction = 1 if @$pcidevices > 1;
3726
3727         my $sysfspath;
3728         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3729             my $pci_id = $pcidevices->[0]->{id};
3730             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3731             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3732         } elsif ($d->{mdev}) {
3733             warn "ignoring mediated device '$id' with multifunction device\n";
3734         }
3735
3736         my $j=0;
3737         foreach my $pcidevice (@$pcidevices) {
3738             my $devicestr = "vfio-pci";
3739
3740             if ($sysfspath) {
3741                 $devicestr .= ",sysfsdev=$sysfspath";
3742             } else {
3743                 $devicestr .= ",host=$pcidevice->{id}";
3744             }
3745
3746             my $mf_addr = $multifunction ? ".$j" : '';
3747             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3748
3749             if ($j == 0) {
3750                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3751                 $devicestr .= "$xvga";
3752                 $devicestr .= ",multifunction=on" if $multifunction;
3753                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3754             }
3755
3756             push @$devices, '-device', $devicestr;
3757             $j++;
3758         }
3759     }
3760
3761     # usb devices
3762     my $usb_dev_features = {};
3763     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3764
3765     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3766     push @$devices, @usbdevices if @usbdevices;
3767     # serial devices
3768     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3769         if (my $path = $conf->{"serial$i"}) {
3770             if ($path eq 'socket') {
3771                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3772                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3773                 # On aarch64, serial0 is the UART device. Qemu only allows
3774                 # connecting UART devices via the '-serial' command line, as
3775                 # the device has a fixed slot on the hardware...
3776                 if ($arch eq 'aarch64' && $i == 0) {
3777                     push @$devices, '-serial', "chardev:serial$i";
3778                 } else {
3779                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3780                 }
3781             } else {
3782                 die "no such serial device\n" if ! -c $path;
3783                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3784                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3785             }
3786         }
3787     }
3788
3789     # parallel devices
3790     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3791         if (my $path = $conf->{"parallel$i"}) {
3792             die "no such parallel device\n" if ! -c $path;
3793             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3794             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3795             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3796         }
3797     }
3798
3799     if (my $audio = conf_has_audio($conf)) {
3800
3801         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3802
3803         my $id = $audio->{dev_id};
3804         if ($audio->{dev} eq 'AC97') {
3805             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3806         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3807             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3808             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3809             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3810         } else {
3811             die "unkown audio device '$audio->{dev}', implement me!";
3812         }
3813
3814         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3815     }
3816
3817     my $sockets = 1;
3818     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3819     $sockets = $conf->{sockets} if  $conf->{sockets};
3820
3821     my $cores = $conf->{cores} || 1;
3822
3823     my $maxcpus = $sockets * $cores;
3824
3825     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3826
3827     my $allowed_vcpus = $cpuinfo->{cpus};
3828
3829     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3830         if ($allowed_vcpus < $maxcpus);
3831
3832     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3833
3834         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3835         for (my $i = 2; $i <= $vcpus; $i++)  {
3836             my $cpustr = print_cpu_device($conf,$i);
3837             push @$cmd, '-device', $cpustr;
3838         }
3839
3840     } else {
3841
3842         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3843     }
3844     push @$cmd, '-nodefaults';
3845
3846     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3847
3848     my $bootindex_hash = {};
3849     my $i = 1;
3850     foreach my $o (split(//, $bootorder)) {
3851         $bootindex_hash->{$o} = $i*100;
3852         $i++;
3853     }
3854
3855     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3856
3857     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3858
3859     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3860
3861     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3862         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3863         my $socket = vnc_socket($vmid);
3864         push @$cmd,  '-vnc', "unix:$socket,password";
3865     } else {
3866         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3867         push @$cmd, '-nographic';
3868     }
3869
3870     # time drift fix
3871     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3872
3873     my $useLocaltime = $conf->{localtime};
3874
3875     if ($winversion >= 5) { # windows
3876         $useLocaltime = 1 if !defined($conf->{localtime});
3877
3878         # use time drift fix when acpi is enabled
3879         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3880             $tdf = 1 if !defined($conf->{tdf});
3881         }
3882     }
3883
3884     if ($winversion >= 6) {
3885         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3886         push @$cmd, '-no-hpet';
3887     }
3888
3889     push @$rtcFlags, 'driftfix=slew' if $tdf;
3890
3891     if (!$kvm) {
3892         push @$machineFlags, 'accel=tcg';
3893     }
3894
3895     if ($machine_type) {
3896         push @$machineFlags, "type=${machine_type}";
3897     }
3898
3899     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3900         push @$rtcFlags, "base=$conf->{startdate}";
3901     } elsif ($useLocaltime) {
3902         push @$rtcFlags, 'base=localtime';
3903     }
3904
3905     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3906
3907     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3908
3909     push @$cmd, '-S' if $conf->{freeze};
3910
3911     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3912
3913     if (parse_guest_agent($conf)->{enabled}) {
3914         my $qgasocket = qmp_socket($vmid, 1);
3915         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3916         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3917         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3918         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3919     }
3920
3921     my $spice_port;
3922
3923     if ($qxlnum) {
3924         if ($qxlnum > 1) {
3925             if ($winversion){
3926                 for(my $i = 1; $i < $qxlnum; $i++){
3927                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3928                 }
3929             } else {
3930                 # assume other OS works like Linux
3931                 my ($ram, $vram) = ("134217728", "67108864");
3932                 if ($vga->{memory}) {
3933                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3934                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3935                 }
3936                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3937                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3938             }
3939         }
3940
3941         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3942
3943         my $nodename = PVE::INotify::nodename();
3944         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3945         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3946         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3947
3948         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3949         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3950         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3951
3952         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3953         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3954
3955         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3956         if ($spice_enhancement->{foldersharing}) {
3957             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3958             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3959         }
3960
3961         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3962         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3963         push @$devices, '-spice', "$spice_opts";
3964     }
3965
3966     # enable balloon by default, unless explicitly disabled
3967     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3968         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3969         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3970     }
3971
3972     if ($conf->{watchdog}) {
3973         my $wdopts = parse_watchdog($conf->{watchdog});
3974         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3975         my $watchdog = $wdopts->{model} || 'i6300esb';
3976         push @$devices, '-device', "$watchdog$pciaddr";
3977         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3978     }
3979
3980     my $vollist = [];
3981     my $scsicontroller = {};
3982     my $ahcicontroller = {};
3983     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3984
3985     # Add iscsi initiator name if available
3986     if (my $initiator = get_initiator_name()) {
3987         push @$devices, '-iscsi', "initiator-name=$initiator";
3988     }
3989
3990     foreach_drive($conf, sub {
3991         my ($ds, $drive) = @_;
3992
3993         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3994             push @$vollist, $drive->{file};
3995         }
3996
3997         # ignore efidisk here, already added in bios/fw handling code above
3998         return if $drive->{interface} eq 'efidisk';
3999
4000         $use_virtio = 1 if $ds =~ m/^virtio/;
4001
4002         if (drive_is_cdrom ($drive)) {
4003             if ($bootindex_hash->{d}) {
4004                 $drive->{bootindex} = $bootindex_hash->{d};
4005                 $bootindex_hash->{d} += 1;
4006             }
4007         } else {
4008             if ($bootindex_hash->{c}) {
4009                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4010                 $bootindex_hash->{c} += 1;
4011             }
4012         }
4013
4014         if($drive->{interface} eq 'virtio'){
4015            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4016         }
4017
4018         if ($drive->{interface} eq 'scsi') {
4019
4020             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4021
4022             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4023             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4024
4025             my $iothread = '';
4026             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4027                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4028                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4029             } elsif ($drive->{iothread}) {
4030                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4031             }
4032
4033             my $queues = '';
4034             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4035                 $queues = ",num_queues=$drive->{queues}";
4036             }
4037
4038             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4039             $scsicontroller->{$controller}=1;
4040         }
4041
4042         if ($drive->{interface} eq 'sata') {
4043            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4044            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4045            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4046            $ahcicontroller->{$controller}=1;
4047         }
4048
4049         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4050         push @$devices, '-drive',$drive_cmd;
4051         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4052     });
4053
4054     for (my $i = 0; $i < $MAX_NETS; $i++) {
4055          next if !$conf->{"net$i"};
4056          my $d = parse_net($conf->{"net$i"});
4057          next if !$d;
4058
4059          $use_virtio = 1 if $d->{model} eq 'virtio';
4060
4061          if ($bootindex_hash->{n}) {
4062             $d->{bootindex} = $bootindex_hash->{n};
4063             $bootindex_hash->{n} += 1;
4064          }
4065
4066          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4067          push @$devices, '-netdev', $netdevfull;
4068
4069          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4070          push @$devices, '-device', $netdevicefull;
4071     }
4072
4073     if ($conf->{ivshmem}) {
4074         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4075
4076         my $bus;
4077         if ($q35) {
4078             $bus = print_pcie_addr("ivshmem");
4079         } else {
4080             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4081         }
4082
4083         my $ivshmem_name = $ivshmem->{name} // $vmid;
4084         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4085
4086         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4087         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4088     }
4089
4090     if (!$q35) {
4091         # add pci bridges
4092         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4093            $bridges->{1} = 1;
4094            $bridges->{2} = 1;
4095         }
4096
4097         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4098
4099         for my $k (sort {$b cmp $a} keys %$bridges) {
4100             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4101             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4102         }
4103     }
4104
4105     push @$cmd, @$devices;
4106     push @$cmd, '-rtc', join(',', @$rtcFlags)
4107         if scalar(@$rtcFlags);
4108     push @$cmd, '-machine', join(',', @$machineFlags)
4109         if scalar(@$machineFlags);
4110     push @$cmd, '-global', join(',', @$globalFlags)
4111         if scalar(@$globalFlags);
4112
4113     if (my $vmstate = $conf->{vmstate}) {
4114         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4115         push @$vollist, $vmstate;
4116         push @$cmd, '-loadstate', $statepath;
4117     }
4118
4119     # add custom args
4120     if ($conf->{args}) {
4121         my $aa = PVE::Tools::split_args($conf->{args});
4122         push @$cmd, @$aa;
4123     }
4124
4125     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4126 }
4127
4128 sub vnc_socket {
4129     my ($vmid) = @_;
4130     return "${var_run_tmpdir}/$vmid.vnc";
4131 }
4132
4133 sub spice_port {
4134     my ($vmid) = @_;
4135
4136     my $res = vm_mon_cmd($vmid, 'query-spice');
4137
4138     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4139 }
4140
4141 sub qmp_socket {
4142     my ($vmid, $qga, $name) = @_;
4143     my $sockettype = $qga ? 'qga' : 'qmp';
4144     my $ext = $name ? '-'.$name : '';
4145     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4146 }
4147
4148 sub pidfile_name {
4149     my ($vmid) = @_;
4150     return "${var_run_tmpdir}/$vmid.pid";
4151 }
4152
4153 sub vm_devices_list {
4154     my ($vmid) = @_;
4155
4156     my $res = vm_mon_cmd($vmid, 'query-pci');
4157     my $devices_to_check = [];
4158     my $devices = {};
4159     foreach my $pcibus (@$res) {
4160         push @$devices_to_check, @{$pcibus->{devices}},
4161     }
4162
4163     while (@$devices_to_check) {
4164         my $to_check = [];
4165         for my $d (@$devices_to_check) {
4166             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4167             next if !$d->{'pci_bridge'};
4168
4169             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4170             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4171         }
4172         $devices_to_check = $to_check;
4173     }
4174
4175     my $resblock = vm_mon_cmd($vmid, 'query-block');
4176     foreach my $block (@$resblock) {
4177         if($block->{device} =~ m/^drive-(\S+)/){
4178                 $devices->{$1} = 1;
4179         }
4180     }
4181
4182     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4183     foreach my $mice (@$resmice) {
4184         if ($mice->{name} eq 'QEMU HID Tablet') {
4185             $devices->{tablet} = 1;
4186             last;
4187         }
4188     }
4189
4190     # for usb devices there is no query-usb
4191     # but we can iterate over the entries in
4192     # qom-list path=/machine/peripheral
4193     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4194     foreach my $per (@$resperipheral) {
4195         if ($per->{name} =~ m/^usb\d+$/) {
4196             $devices->{$per->{name}} = 1;
4197         }
4198     }
4199
4200     return $devices;
4201 }
4202
4203 sub vm_deviceplug {
4204     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4205
4206     my $q35 = machine_type_is_q35($conf);
4207
4208     my $devices_list = vm_devices_list($vmid);
4209     return 1 if defined($devices_list->{$deviceid});
4210
4211     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4212
4213     if ($deviceid eq 'tablet') {
4214
4215         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4216
4217     } elsif ($deviceid eq 'keyboard') {
4218
4219         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4220
4221     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4222
4223         die "usb hotplug currently not reliable\n";
4224         # since we can't reliably hot unplug all added usb devices
4225         # and usb passthrough disables live migration
4226         # we disable usb hotplugging for now
4227         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4228
4229     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4230
4231         qemu_iothread_add($vmid, $deviceid, $device);
4232
4233         qemu_driveadd($storecfg, $vmid, $device);
4234         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4235
4236         qemu_deviceadd($vmid, $devicefull);
4237         eval { qemu_deviceaddverify($vmid, $deviceid); };
4238         if (my $err = $@) {
4239             eval { qemu_drivedel($vmid, $deviceid); };
4240             warn $@ if $@;
4241             die $err;
4242         }
4243
4244     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4245
4246
4247         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4248         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4249         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4250
4251         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4252
4253         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4254             qemu_iothread_add($vmid, $deviceid, $device);
4255             $devicefull .= ",iothread=iothread-$deviceid";
4256         }
4257
4258         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4259             $devicefull .= ",num_queues=$device->{queues}";
4260         }
4261
4262         qemu_deviceadd($vmid, $devicefull);
4263         qemu_deviceaddverify($vmid, $deviceid);
4264
4265     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4266
4267         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4268         qemu_driveadd($storecfg, $vmid, $device);
4269
4270         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4271         eval { qemu_deviceadd($vmid, $devicefull); };
4272         if (my $err = $@) {
4273             eval { qemu_drivedel($vmid, $deviceid); };
4274             warn $@ if $@;
4275             die $err;
4276         }
4277
4278     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4279
4280         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4281
4282         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4283         my $use_old_bios_files = undef;
4284         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4285
4286         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4287         qemu_deviceadd($vmid, $netdevicefull);
4288         eval {
4289             qemu_deviceaddverify($vmid, $deviceid);
4290             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4291         };
4292         if (my $err = $@) {
4293             eval { qemu_netdevdel($vmid, $deviceid); };
4294             warn $@ if $@;
4295             die $err;
4296         }
4297
4298     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4299
4300         my $bridgeid = $2;
4301         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4302         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4303
4304         qemu_deviceadd($vmid, $devicefull);
4305         qemu_deviceaddverify($vmid, $deviceid);
4306
4307     } else {
4308         die "can't hotplug device '$deviceid'\n";
4309     }
4310
4311     return 1;
4312 }
4313
4314 # fixme: this should raise exceptions on error!
4315 sub vm_deviceunplug {
4316     my ($vmid, $conf, $deviceid) = @_;
4317
4318     my $devices_list = vm_devices_list($vmid);
4319     return 1 if !defined($devices_list->{$deviceid});
4320
4321     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4322
4323     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4324
4325         qemu_devicedel($vmid, $deviceid);
4326
4327     } elsif ($deviceid =~ m/^usb\d+$/) {
4328
4329         die "usb hotplug currently not reliable\n";
4330         # when unplugging usb devices this way,
4331         # there may be remaining usb controllers/hubs
4332         # so we disable it for now
4333         qemu_devicedel($vmid, $deviceid);
4334         qemu_devicedelverify($vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4337
4338         qemu_devicedel($vmid, $deviceid);
4339         qemu_devicedelverify($vmid, $deviceid);
4340         qemu_drivedel($vmid, $deviceid);
4341         qemu_iothread_del($conf, $vmid, $deviceid);
4342
4343     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4344
4345         qemu_devicedel($vmid, $deviceid);
4346         qemu_devicedelverify($vmid, $deviceid);
4347         qemu_iothread_del($conf, $vmid, $deviceid);
4348
4349     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4350
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_drivedel($vmid, $deviceid);
4353         qemu_deletescsihw($conf, $vmid, $deviceid);
4354
4355     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4356
4357         qemu_devicedel($vmid, $deviceid);
4358         qemu_devicedelverify($vmid, $deviceid);
4359         qemu_netdevdel($vmid, $deviceid);
4360
4361     } else {
4362         die "can't unplug device '$deviceid'\n";
4363     }
4364
4365     return 1;
4366 }
4367
4368 sub qemu_deviceadd {
4369     my ($vmid, $devicefull) = @_;
4370
4371     $devicefull = "driver=".$devicefull;
4372     my %options =  split(/[=,]/, $devicefull);
4373
4374     vm_mon_cmd($vmid, "device_add" , %options);
4375 }
4376
4377 sub qemu_devicedel {
4378     my ($vmid, $deviceid) = @_;
4379
4380     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4381 }
4382
4383 sub qemu_iothread_add {
4384     my($vmid, $deviceid, $device) = @_;
4385
4386     if ($device->{iothread}) {
4387         my $iothreads = vm_iothreads_list($vmid);
4388         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4389     }
4390 }
4391
4392 sub qemu_iothread_del {
4393     my($conf, $vmid, $deviceid) = @_;
4394
4395     my $confid = $deviceid;
4396     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4397         $confid = 'scsi' . $1;
4398     }
4399     my $device = parse_drive($confid, $conf->{$confid});
4400     if ($device->{iothread}) {
4401         my $iothreads = vm_iothreads_list($vmid);
4402         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4403     }
4404 }
4405
4406 sub qemu_objectadd {
4407     my($vmid, $objectid, $qomtype) = @_;
4408
4409     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4410
4411     return 1;
4412 }
4413
4414 sub qemu_objectdel {
4415     my($vmid, $objectid) = @_;
4416
4417     vm_mon_cmd($vmid, "object-del", id => $objectid);
4418
4419     return 1;
4420 }
4421
4422 sub qemu_driveadd {
4423     my ($storecfg, $vmid, $device) = @_;
4424
4425     my $drive = print_drive_full($storecfg, $vmid, $device);
4426     $drive =~ s/\\/\\\\/g;
4427     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4428
4429     # If the command succeeds qemu prints: "OK"
4430     return 1 if $ret =~ m/OK/s;
4431
4432     die "adding drive failed: $ret\n";
4433 }
4434
4435 sub qemu_drivedel {
4436     my($vmid, $deviceid) = @_;
4437
4438     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4439     $ret =~ s/^\s+//;
4440
4441     return 1 if $ret eq "";
4442
4443     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4444     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4445
4446     die "deleting drive $deviceid failed : $ret\n";
4447 }
4448
4449 sub qemu_deviceaddverify {
4450     my ($vmid, $deviceid) = @_;
4451
4452     for (my $i = 0; $i <= 5; $i++) {
4453          my $devices_list = vm_devices_list($vmid);
4454          return 1 if defined($devices_list->{$deviceid});
4455          sleep 1;
4456     }
4457
4458     die "error on hotplug device '$deviceid'\n";
4459 }
4460
4461
4462 sub qemu_devicedelverify {
4463     my ($vmid, $deviceid) = @_;
4464
4465     # need to verify that the device is correctly removed as device_del
4466     # is async and empty return is not reliable
4467
4468     for (my $i = 0; $i <= 5; $i++) {
4469          my $devices_list = vm_devices_list($vmid);
4470          return 1 if !defined($devices_list->{$deviceid});
4471          sleep 1;
4472     }
4473
4474     die "error on hot-unplugging device '$deviceid'\n";
4475 }
4476
4477 sub qemu_findorcreatescsihw {
4478     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4479
4480     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4481
4482     my $scsihwid="$controller_prefix$controller";
4483     my $devices_list = vm_devices_list($vmid);
4484
4485     if(!defined($devices_list->{$scsihwid})) {
4486         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4487     }
4488
4489     return 1;
4490 }
4491
4492 sub qemu_deletescsihw {
4493     my ($conf, $vmid, $opt) = @_;
4494
4495     my $device = parse_drive($opt, $conf->{$opt});
4496
4497     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4498         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4499         return 1;
4500     }
4501
4502     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4503
4504     my $devices_list = vm_devices_list($vmid);
4505     foreach my $opt (keys %{$devices_list}) {
4506         if (PVE::QemuServer::is_valid_drivename($opt)) {
4507             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4508             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4509                 return 1;
4510             }
4511         }
4512     }
4513
4514     my $scsihwid="scsihw$controller";
4515
4516     vm_deviceunplug($vmid, $conf, $scsihwid);
4517
4518     return 1;
4519 }
4520
4521 sub qemu_add_pci_bridge {
4522     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4523
4524     my $bridges = {};
4525
4526     my $bridgeid;
4527
4528     print_pci_addr($device, $bridges, $arch, $machine_type);
4529
4530     while (my ($k, $v) = each %$bridges) {
4531         $bridgeid = $k;
4532     }
4533     return 1 if !defined($bridgeid) || $bridgeid < 1;
4534
4535     my $bridge = "pci.$bridgeid";
4536     my $devices_list = vm_devices_list($vmid);
4537
4538     if (!defined($devices_list->{$bridge})) {
4539         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4540     }
4541
4542     return 1;
4543 }
4544
4545 sub qemu_set_link_status {
4546     my ($vmid, $device, $up) = @_;
4547
4548     vm_mon_cmd($vmid, "set_link", name => $device,
4549                up => $up ? JSON::true : JSON::false);
4550 }
4551
4552 sub qemu_netdevadd {
4553     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4554
4555     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4556     my %options =  split(/[=,]/, $netdev);
4557
4558     vm_mon_cmd($vmid, "netdev_add",  %options);
4559     return 1;
4560 }
4561
4562 sub qemu_netdevdel {
4563     my ($vmid, $deviceid) = @_;
4564
4565     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4566 }
4567
4568 sub qemu_usb_hotplug {
4569     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4570
4571     return if !$device;
4572
4573     # remove the old one first
4574     vm_deviceunplug($vmid, $conf, $deviceid);
4575
4576     # check if xhci controller is necessary and available
4577     if ($device->{usb3}) {
4578
4579         my $devicelist = vm_devices_list($vmid);
4580
4581         if (!$devicelist->{xhci}) {
4582             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4583             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4584         }
4585     }
4586     my $d = parse_usb_device($device->{host});
4587     $d->{usb3} = $device->{usb3};
4588
4589     # add the new one
4590     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4591 }
4592
4593 sub qemu_cpu_hotplug {
4594     my ($vmid, $conf, $vcpus) = @_;
4595
4596     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4597
4598     my $sockets = 1;
4599     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4600     $sockets = $conf->{sockets} if  $conf->{sockets};
4601     my $cores = $conf->{cores} || 1;
4602     my $maxcpus = $sockets * $cores;
4603
4604     $vcpus = $maxcpus if !$vcpus;
4605
4606     die "you can't add more vcpus than maxcpus\n"
4607         if $vcpus > $maxcpus;
4608
4609     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4610
4611     if ($vcpus < $currentvcpus) {
4612
4613         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4614
4615             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4616                 qemu_devicedel($vmid, "cpu$i");
4617                 my $retry = 0;
4618                 my $currentrunningvcpus = undef;
4619                 while (1) {
4620                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4621                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4622                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4623                     $retry++;
4624                     sleep 1;
4625                 }
4626                 #update conf after each succesfull cpu unplug
4627                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4628                 PVE::QemuConfig->write_config($vmid, $conf);
4629             }
4630         } else {
4631             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4632         }
4633
4634         return;
4635     }
4636
4637     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4638     die "vcpus in running vm does not match its configuration\n"
4639         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4640
4641     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4642
4643         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4644             my $cpustr = print_cpu_device($conf, $i);
4645             qemu_deviceadd($vmid, $cpustr);
4646
4647             my $retry = 0;
4648             my $currentrunningvcpus = undef;
4649             while (1) {
4650                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4651                 last if scalar(@{$currentrunningvcpus}) == $i;
4652                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4653                 sleep 1;
4654                 $retry++;
4655             }
4656             #update conf after each succesfull cpu hotplug
4657             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4658             PVE::QemuConfig->write_config($vmid, $conf);
4659         }
4660     } else {
4661
4662         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4663             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4664         }
4665     }
4666 }
4667
4668 sub qemu_block_set_io_throttle {
4669     my ($vmid, $deviceid,
4670         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4671         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4672         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4673         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4674
4675     return if !check_running($vmid) ;
4676
4677     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4678         bps => int($bps),
4679         bps_rd => int($bps_rd),
4680         bps_wr => int($bps_wr),
4681         iops => int($iops),
4682         iops_rd => int($iops_rd),
4683         iops_wr => int($iops_wr),
4684         bps_max => int($bps_max),
4685         bps_rd_max => int($bps_rd_max),
4686         bps_wr_max => int($bps_wr_max),
4687         iops_max => int($iops_max),
4688         iops_rd_max => int($iops_rd_max),
4689         iops_wr_max => int($iops_wr_max),
4690         bps_max_length => int($bps_max_length),
4691         bps_rd_max_length => int($bps_rd_max_length),
4692         bps_wr_max_length => int($bps_wr_max_length),
4693         iops_max_length => int($iops_max_length),
4694         iops_rd_max_length => int($iops_rd_max_length),
4695         iops_wr_max_length => int($iops_wr_max_length),
4696     );
4697
4698 }
4699
4700 # old code, only used to shutdown old VM after update
4701 sub __read_avail {
4702     my ($fh, $timeout) = @_;
4703
4704     my $sel = new IO::Select;
4705     $sel->add($fh);
4706
4707     my $res = '';
4708     my $buf;
4709
4710     my @ready;
4711     while (scalar (@ready = $sel->can_read($timeout))) {
4712         my $count;
4713         if ($count = $fh->sysread($buf, 8192)) {
4714             if ($buf =~ /^(.*)\(qemu\) $/s) {
4715                 $res .= $1;
4716                 last;
4717             } else {
4718                 $res .= $buf;
4719             }
4720         } else {
4721             if (!defined($count)) {
4722                 die "$!\n";
4723             }
4724             last;
4725         }
4726     }
4727
4728     die "monitor read timeout\n" if !scalar(@ready);
4729
4730     return $res;
4731 }
4732
4733 sub qemu_block_resize {
4734     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4735
4736     my $running = check_running($vmid);
4737
4738     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4739
4740     return if !$running;
4741
4742     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4743
4744 }
4745
4746 sub qemu_volume_snapshot {
4747     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4748
4749     my $running = check_running($vmid);
4750
4751     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4752         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4753     } else {
4754         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4755     }
4756 }
4757
4758 sub qemu_volume_snapshot_delete {
4759     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4760
4761     my $running = check_running($vmid);
4762
4763     if($running) {
4764
4765         $running = undef;
4766         my $conf = PVE::QemuConfig->load_config($vmid);
4767         foreach_drive($conf, sub {
4768             my ($ds, $drive) = @_;
4769             $running = 1 if $drive->{file} eq $volid;
4770         });
4771     }
4772
4773     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4774         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4775     } else {
4776         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4777     }
4778 }
4779
4780 sub set_migration_caps {
4781     my ($vmid) = @_;
4782
4783     my $cap_ref = [];
4784
4785     my $enabled_cap = {
4786         "auto-converge" => 1,
4787         "xbzrle" => 1,
4788         "x-rdma-pin-all" => 0,
4789         "zero-blocks" => 0,
4790         "compress" => 0
4791     };
4792
4793     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4794
4795     for my $supported_capability (@$supported_capabilities) {
4796         push @$cap_ref, {
4797             capability => $supported_capability->{capability},
4798             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4799         };
4800     }
4801
4802     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4803 }
4804
4805 my $fast_plug_option = {
4806     'lock' => 1,
4807     'name' => 1,
4808     'onboot' => 1,
4809     'shares' => 1,
4810     'startup' => 1,
4811     'description' => 1,
4812     'protection' => 1,
4813     'vmstatestorage' => 1,
4814     'hookscript' => 1,
4815 };
4816
4817 # hotplug changes in [PENDING]
4818 # $selection hash can be used to only apply specified options, for
4819 # example: { cores => 1 } (only apply changed 'cores')
4820 # $errors ref is used to return error messages
4821 sub vmconfig_hotplug_pending {
4822     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4823
4824     my $defaults = load_defaults();
4825     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4826
4827     # commit values which do not have any impact on running VM first
4828     # Note: those option cannot raise errors, we we do not care about
4829     # $selection and always apply them.
4830
4831     my $add_error = sub {
4832         my ($opt, $msg) = @_;
4833         $errors->{$opt} = "hotplug problem - $msg";
4834     };
4835
4836     my $changes = 0;
4837     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4838         if ($fast_plug_option->{$opt}) {
4839             $conf->{$opt} = $conf->{pending}->{$opt};
4840             delete $conf->{pending}->{$opt};
4841             $changes = 1;
4842         }
4843     }
4844
4845     if ($changes) {
4846         PVE::QemuConfig->write_config($vmid, $conf);
4847         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4848     }
4849
4850     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4851
4852     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4853     foreach my $opt (sort keys %$pending_delete_hash) {
4854         next if $selection && !$selection->{$opt};
4855         my $force = $pending_delete_hash->{$opt}->{force};
4856         eval {
4857             if ($opt eq 'hotplug') {
4858                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4859             } elsif ($opt eq 'tablet') {
4860                 die "skip\n" if !$hotplug_features->{usb};
4861                 if ($defaults->{tablet}) {
4862                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4863                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4864                         if $arch eq 'aarch64';
4865                 } else {
4866                     vm_deviceunplug($vmid, $conf, 'tablet');
4867                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4868                 }
4869             } elsif ($opt =~ m/^usb\d+/) {
4870                 die "skip\n";
4871                 # since we cannot reliably hot unplug usb devices
4872                 # we are disabling it
4873                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4874                 vm_deviceunplug($vmid, $conf, $opt);
4875             } elsif ($opt eq 'vcpus') {
4876                 die "skip\n" if !$hotplug_features->{cpu};
4877                 qemu_cpu_hotplug($vmid, $conf, undef);
4878             } elsif ($opt eq 'balloon') {
4879                 # enable balloon device is not hotpluggable
4880                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4881                 # here we reset the ballooning value to memory
4882                 my $balloon = $conf->{memory} || $defaults->{memory};
4883                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4884             } elsif ($fast_plug_option->{$opt}) {
4885                 # do nothing
4886             } elsif ($opt =~ m/^net(\d+)$/) {
4887                 die "skip\n" if !$hotplug_features->{network};
4888                 vm_deviceunplug($vmid, $conf, $opt);
4889             } elsif (is_valid_drivename($opt)) {
4890                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4891                 vm_deviceunplug($vmid, $conf, $opt);
4892                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4893             } elsif ($opt =~ m/^memory$/) {
4894                 die "skip\n" if !$hotplug_features->{memory};
4895                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4896             } elsif ($opt eq 'cpuunits') {
4897                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4898             } elsif ($opt eq 'cpulimit') {
4899                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4900             } else {
4901                 die "skip\n";
4902             }
4903         };
4904         if (my $err = $@) {
4905             &$add_error($opt, $err) if $err ne "skip\n";
4906         } else {
4907             # save new config if hotplug was successful
4908             delete $conf->{$opt};
4909             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4910             PVE::QemuConfig->write_config($vmid, $conf);
4911             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4912         }
4913     }
4914
4915     my $apply_pending_cloudinit;
4916     $apply_pending_cloudinit = sub {
4917         my ($key, $value) = @_;
4918         $apply_pending_cloudinit = sub {}; # once is enough
4919
4920         my @cloudinit_opts = keys %$confdesc_cloudinit;
4921         foreach my $opt (keys %{$conf->{pending}}) {
4922             next if !grep { $_ eq $opt } @cloudinit_opts;
4923             $conf->{$opt} = delete $conf->{pending}->{$opt};
4924         }
4925
4926         my $new_conf = { %$conf };
4927         $new_conf->{$key} = $value;
4928         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4929     };
4930
4931     foreach my $opt (keys %{$conf->{pending}}) {
4932         next if $selection && !$selection->{$opt};
4933         my $value = $conf->{pending}->{$opt};
4934         eval {
4935             if ($opt eq 'hotplug') {
4936                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4937             } elsif ($opt eq 'tablet') {
4938                 die "skip\n" if !$hotplug_features->{usb};
4939                 if ($value == 1) {
4940                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4941                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4942                         if $arch eq 'aarch64';
4943                 } elsif ($value == 0) {
4944                     vm_deviceunplug($vmid, $conf, 'tablet');
4945                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4946                 }
4947             } elsif ($opt =~ m/^usb\d+$/) {
4948                 die "skip\n";
4949                 # since we cannot reliably hot unplug usb devices
4950                 # we are disabling it
4951                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4952                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4953                 die "skip\n" if !$d;
4954                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, $value);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable/disable balloning device is not hotpluggable
4960                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4961                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4962                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4963
4964                 # allow manual ballooning if shares is set to zero
4965                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4966                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4967                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4968                 }
4969             } elsif ($opt =~ m/^net(\d+)$/) {
4970                 # some changes can be done without hotplug
4971                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4972                                     $vmid, $opt, $value, $arch, $machine_type);
4973             } elsif (is_valid_drivename($opt)) {
4974                 # some changes can be done without hotplug
4975                 my $drive = parse_drive($opt, $value);
4976                 if (drive_is_cloudinit($drive)) {
4977                     &$apply_pending_cloudinit($opt, $value);
4978                 }
4979                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4980                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4981             } elsif ($opt =~ m/^memory$/) { #dimms
4982                 die "skip\n" if !$hotplug_features->{memory};
4983                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4984             } elsif ($opt eq 'cpuunits') {
4985                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4986             } elsif ($opt eq 'cpulimit') {
4987                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4988                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4989             } else {
4990                 die "skip\n";  # skip non-hot-pluggable options
4991             }
4992         };
4993         if (my $err = $@) {
4994             &$add_error($opt, $err) if $err ne "skip\n";
4995         } else {
4996             # save new config if hotplug was successful
4997             $conf->{$opt} = $value;
4998             delete $conf->{pending}->{$opt};
4999             PVE::QemuConfig->write_config($vmid, $conf);
5000             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5001         }
5002     }
5003 }
5004
5005 sub try_deallocate_drive {
5006     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5007
5008     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5009         my $volid = $drive->{file};
5010         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5011             my $sid = PVE::Storage::parse_volume_id($volid);
5012             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5013
5014             # check if the disk is really unused
5015             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5016                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5017             PVE::Storage::vdisk_free($storecfg, $volid);
5018             return 1;
5019         } else {
5020             # If vm is not owner of this disk remove from config
5021             return 1;
5022         }
5023     }
5024
5025     return undef;
5026 }
5027
5028 sub vmconfig_delete_or_detach_drive {
5029     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5030
5031     my $drive = parse_drive($opt, $conf->{$opt});
5032
5033     my $rpcenv = PVE::RPCEnvironment::get();
5034     my $authuser = $rpcenv->get_user();
5035
5036     if ($force) {
5037         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5038         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5039     } else {
5040         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5041     }
5042 }
5043
5044
5045
5046 sub vmconfig_apply_pending {
5047     my ($vmid, $conf, $storecfg) = @_;
5048
5049     # cold plug
5050
5051     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5052     foreach my $opt (sort keys %$pending_delete_hash) {
5053         die "internal error" if $opt =~ m/^unused/;
5054         my $force = $pending_delete_hash->{$opt}->{force};
5055         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5056         if (!defined($conf->{$opt})) {
5057             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5058             PVE::QemuConfig->write_config($vmid, $conf);
5059         } elsif (is_valid_drivename($opt)) {
5060             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5061             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5062             delete $conf->{$opt};
5063             PVE::QemuConfig->write_config($vmid, $conf);
5064         } else {
5065             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5066             delete $conf->{$opt};
5067             PVE::QemuConfig->write_config($vmid, $conf);
5068         }
5069     }
5070
5071     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5072
5073     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5074         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5075
5076         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5077             # skip if nothing changed
5078         } elsif (is_valid_drivename($opt)) {
5079             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5080                 if defined($conf->{$opt});
5081             $conf->{$opt} = $conf->{pending}->{$opt};
5082         } else {
5083             $conf->{$opt} = $conf->{pending}->{$opt};
5084         }
5085
5086         delete $conf->{pending}->{$opt};
5087         PVE::QemuConfig->write_config($vmid, $conf);
5088     }
5089 }
5090
5091 my $safe_num_ne = sub {
5092     my ($a, $b) = @_;
5093
5094     return 0 if !defined($a) && !defined($b);
5095     return 1 if !defined($a);
5096     return 1 if !defined($b);
5097
5098     return $a != $b;
5099 };
5100
5101 my $safe_string_ne = sub {
5102     my ($a, $b) = @_;
5103
5104     return 0 if !defined($a) && !defined($b);
5105     return 1 if !defined($a);
5106     return 1 if !defined($b);
5107
5108     return $a ne $b;
5109 };
5110
5111 sub vmconfig_update_net {
5112     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5113
5114     my $newnet = parse_net($value);
5115
5116     if ($conf->{$opt}) {
5117         my $oldnet = parse_net($conf->{$opt});
5118
5119         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5120             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5121             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5122             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5123
5124             # for non online change, we try to hot-unplug
5125             die "skip\n" if !$hotplug;
5126             vm_deviceunplug($vmid, $conf, $opt);
5127         } else {
5128
5129             die "internal error" if $opt !~ m/net(\d+)/;
5130             my $iface = "tap${vmid}i$1";
5131
5132             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5133                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5134                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5135                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5136                 PVE::Network::tap_unplug($iface);
5137                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5138             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5139                 # Rate can be applied on its own but any change above needs to
5140                 # include the rate in tap_plug since OVS resets everything.
5141                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5142             }
5143
5144             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5145                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5146             }
5147
5148             return 1;
5149         }
5150     }
5151
5152     if ($hotplug) {
5153         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5154     } else {
5155         die "skip\n";
5156     }
5157 }
5158
5159 sub vmconfig_update_disk {
5160     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5161
5162     # fixme: do we need force?
5163
5164     my $drive = parse_drive($opt, $value);
5165
5166     if ($conf->{$opt}) {
5167
5168         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5169
5170             my $media = $drive->{media} || 'disk';
5171             my $oldmedia = $old_drive->{media} || 'disk';
5172             die "unable to change media type\n" if $media ne $oldmedia;
5173
5174             if (!drive_is_cdrom($old_drive)) {
5175
5176                 if ($drive->{file} ne $old_drive->{file}) {
5177
5178                     die "skip\n" if !$hotplug;
5179
5180                     # unplug and register as unused
5181                     vm_deviceunplug($vmid, $conf, $opt);
5182                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5183
5184                 } else {
5185                     # update existing disk
5186
5187                     # skip non hotpluggable value
5188                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5189                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5190                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5191                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5192                         die "skip\n";
5193                     }
5194
5195                     # apply throttle
5196                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5197                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5198                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5199                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5200                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5201                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5202                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5203                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5204                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5205                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5206                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5207                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5208                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5209                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5210                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5211                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5212                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5213                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5214
5215                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5216                                                    ($drive->{mbps} || 0)*1024*1024,
5217                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5218                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5219                                                    $drive->{iops} || 0,
5220                                                    $drive->{iops_rd} || 0,
5221                                                    $drive->{iops_wr} || 0,
5222                                                    ($drive->{mbps_max} || 0)*1024*1024,
5223                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5224                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5225                                                    $drive->{iops_max} || 0,
5226                                                    $drive->{iops_rd_max} || 0,
5227                                                    $drive->{iops_wr_max} || 0,
5228                                                    $drive->{bps_max_length} || 1,
5229                                                    $drive->{bps_rd_max_length} || 1,
5230                                                    $drive->{bps_wr_max_length} || 1,
5231                                                    $drive->{iops_max_length} || 1,
5232                                                    $drive->{iops_rd_max_length} || 1,
5233                                                    $drive->{iops_wr_max_length} || 1);
5234
5235                     }
5236
5237                     return 1;
5238                 }
5239
5240             } else { # cdrom
5241
5242                 if ($drive->{file} eq 'none') {
5243                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5244                     if (drive_is_cloudinit($old_drive)) {
5245                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5246                     }
5247                 } else {
5248                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5249                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5250                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5251                 }
5252
5253                 return 1;
5254             }
5255         }
5256     }
5257
5258     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5259     # hotplug new disks
5260     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5261     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5262 }
5263
5264 sub vm_start {
5265     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5266         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5267
5268     PVE::QemuConfig->lock_config($vmid, sub {
5269         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5270
5271         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5272
5273         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5274
5275         PVE::QemuConfig->check_lock($conf)
5276             if !($skiplock || $is_suspended);
5277
5278         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5279
5280         # clean up leftover reboot request files
5281         eval { clear_reboot_request($vmid); };
5282         warn $@ if $@;
5283
5284         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5285             vmconfig_apply_pending($vmid, $conf, $storecfg);
5286             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5287         }
5288
5289         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5290
5291         my $defaults = load_defaults();
5292
5293         # set environment variable useful inside network script
5294         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5295
5296         my $local_volumes = {};
5297
5298         if ($targetstorage) {
5299             foreach_drive($conf, sub {
5300                 my ($ds, $drive) = @_;
5301
5302                 return if drive_is_cdrom($drive);
5303
5304                 my $volid = $drive->{file};
5305
5306                 return if !$volid;
5307
5308                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5309
5310                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5311                 return if $scfg->{shared};
5312                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5313             });
5314
5315             my $format = undef;
5316
5317             foreach my $opt (sort keys %$local_volumes) {
5318
5319                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5320                 my $drive = parse_drive($opt, $conf->{$opt});
5321
5322                 #if remote storage is specified, use default format
5323                 if ($targetstorage && $targetstorage ne "1") {
5324                     $storeid = $targetstorage;
5325                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5326                     $format = $defFormat;
5327                 } else {
5328                     #else we use same format than original
5329                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5330                     $format = qemu_img_format($scfg, $volid);
5331                 }
5332
5333                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5334                 my $newdrive = $drive;
5335                 $newdrive->{format} = $format;
5336                 $newdrive->{file} = $newvolid;
5337                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5338                 $local_volumes->{$opt} = $drivestr;
5339                 #pass drive to conf for command line
5340                 $conf->{$opt} = $drivestr;
5341             }
5342         }
5343
5344         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5345
5346         if ($is_suspended) {
5347             # enforce machine type on suspended vm to ensure HW compatibility
5348             $forcemachine = $conf->{runningmachine};
5349             print "Resuming suspended VM\n";
5350         }
5351
5352         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5353
5354         my $migrate_uri;
5355         if ($statefile) {
5356             if ($statefile eq 'tcp') {
5357                 my $localip = "localhost";
5358                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5359                 my $nodename = PVE::INotify::nodename();
5360
5361                 if (!defined($migration_type)) {
5362                     if (defined($datacenterconf->{migration}->{type})) {
5363                         $migration_type = $datacenterconf->{migration}->{type};
5364                     } else {
5365                         $migration_type = 'secure';
5366                     }
5367                 }
5368
5369                 if ($migration_type eq 'insecure') {
5370                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5371                     if ($migrate_network_addr) {
5372                         $localip = $migrate_network_addr;
5373                     } else {
5374                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5375                     }
5376
5377                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5378                 }
5379
5380                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5381                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5382                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5383                 push @$cmd, '-incoming', $migrate_uri;
5384                 push @$cmd, '-S';
5385
5386             } elsif ($statefile eq 'unix') {
5387                 # should be default for secure migrations as a ssh TCP forward
5388                 # tunnel is not deterministic reliable ready and fails regurarly
5389                 # to set up in time, so use UNIX socket forwards
5390                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5391                 unlink $socket_addr;
5392
5393                 $migrate_uri = "unix:$socket_addr";
5394
5395                 push @$cmd, '-incoming', $migrate_uri;
5396                 push @$cmd, '-S';
5397
5398             } elsif (-e $statefile) {
5399                 push @$cmd, '-loadstate', $statefile;
5400             } else {
5401                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5402                 push @$vollist, $statefile;
5403                 push @$cmd, '-loadstate', $statepath;
5404             }
5405         } elsif ($paused) {
5406             push @$cmd, '-S';
5407         }
5408
5409         # host pci devices
5410         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5411           my $d = parse_hostpci($conf->{"hostpci$i"});
5412           next if !$d;
5413           my $pcidevices = $d->{pciid};
5414           foreach my $pcidevice (@$pcidevices) {
5415                 my $pciid = $pcidevice->{id};
5416
5417                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5418                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5419                 die "no pci device info for device '$pciid'\n" if !$info;
5420
5421                 if ($d->{mdev}) {
5422                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5423                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5424                 } else {
5425                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5426                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5427                     die "can't reset pci device '$pciid'\n"
5428                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5429                 }
5430           }
5431         }
5432
5433         PVE::Storage::activate_volumes($storecfg, $vollist);
5434
5435         eval {
5436             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5437                 outfunc => sub {}, errfunc => sub {});
5438         };
5439         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5440         # timeout should be more than enough here...
5441         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5442
5443         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5444                                                   : $defaults->{cpuunits};
5445
5446         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5447         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5448
5449         my %properties = (
5450             Slice => 'qemu.slice',
5451             KillMode => 'none',
5452             CPUShares => $cpuunits
5453         );
5454
5455         if (my $cpulimit = $conf->{cpulimit}) {
5456             $properties{CPUQuota} = int($cpulimit * 100);
5457         }
5458         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5459
5460         my $run_qemu = sub {
5461             PVE::Tools::run_fork sub {
5462                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5463                 run_command($cmd, %run_params);
5464             };
5465         };
5466
5467         if ($conf->{hugepages}) {
5468
5469             my $code = sub {
5470                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5471                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5472
5473                 PVE::QemuServer::Memory::hugepages_mount();
5474                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5475
5476                 eval { $run_qemu->() };
5477                 if (my $err = $@) {
5478                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5479                     die $err;
5480                 }
5481
5482                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5483             };
5484             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5485
5486         } else {
5487             eval { $run_qemu->() };
5488         }
5489
5490         if (my $err = $@) {
5491             # deactivate volumes if start fails
5492             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5493             die "start failed: $err";
5494         }
5495
5496         print "migration listens on $migrate_uri\n" if $migrate_uri;
5497
5498         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5499             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5500             warn $@ if $@;
5501         }
5502
5503         #start nbd server for storage migration
5504         if ($targetstorage) {
5505             my $nodename = PVE::INotify::nodename();
5506             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5507             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5508             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5509             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5510
5511             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5512
5513             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5514
5515             foreach my $opt (sort keys %$local_volumes) {
5516                 my $volid = $local_volumes->{$opt};
5517                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5518                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5519                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5520             }
5521         }
5522
5523         if ($migratedfrom) {
5524             eval {
5525                 set_migration_caps($vmid);
5526             };
5527             warn $@ if $@;
5528
5529             if ($spice_port) {
5530                 print "spice listens on port $spice_port\n";
5531                 if ($spice_ticket) {
5532                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5533                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5534                 }
5535             }
5536
5537         } else {
5538             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5539                 if !$statefile && $conf->{balloon};
5540
5541             foreach my $opt (keys %$conf) {
5542                 next if $opt !~  m/^net\d+$/;
5543                 my $nicconf = parse_net($conf->{$opt});
5544                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5545             }
5546         }
5547
5548         vm_mon_cmd_nocheck($vmid, 'qom-set',
5549                     path => "machine/peripheral/balloon0",
5550                     property => "guest-stats-polling-interval",
5551                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5552
5553         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5554             print "Resumed VM, removing state\n";
5555             delete $conf->@{qw(lock vmstate runningmachine)};
5556             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5557             PVE::Storage::vdisk_free($storecfg, $vmstate);
5558             PVE::QemuConfig->write_config($vmid, $conf);
5559         }
5560
5561         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5562     });
5563 }
5564
5565 sub vm_mon_cmd {
5566     my ($vmid, $execute, %params) = @_;
5567
5568     my $cmd = { execute => $execute, arguments => \%params };
5569     vm_qmp_command($vmid, $cmd);
5570 }
5571
5572 sub vm_mon_cmd_nocheck {
5573     my ($vmid, $execute, %params) = @_;
5574
5575     my $cmd = { execute => $execute, arguments => \%params };
5576     vm_qmp_command($vmid, $cmd, 1);
5577 }
5578
5579 sub vm_qmp_command {
5580     my ($vmid, $cmd, $nocheck) = @_;
5581
5582     my $res;
5583
5584     my $timeout;
5585     if ($cmd->{arguments}) {
5586         $timeout = delete $cmd->{arguments}->{timeout};
5587     }
5588
5589     eval {
5590         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5591         my $sname = qmp_socket($vmid);
5592         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5593             my $qmpclient = PVE::QMPClient->new();
5594
5595             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5596         } else {
5597             die "unable to open monitor socket\n";
5598         }
5599     };
5600     if (my $err = $@) {
5601         syslog("err", "VM $vmid qmp command failed - $err");
5602         die $err;
5603     }
5604
5605     return $res;
5606 }
5607
5608 sub vm_human_monitor_command {
5609     my ($vmid, $cmdline) = @_;
5610
5611     my $cmd = {
5612         execute => 'human-monitor-command',
5613         arguments => { 'command-line' => $cmdline},
5614     };
5615
5616     return vm_qmp_command($vmid, $cmd);
5617 }
5618
5619 sub vm_commandline {
5620     my ($storecfg, $vmid, $snapname) = @_;
5621
5622     my $conf = PVE::QemuConfig->load_config($vmid);
5623
5624     if ($snapname) {
5625         my $snapshot = $conf->{snapshots}->{$snapname};
5626         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5627
5628         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5629
5630         $conf = $snapshot;
5631     }
5632
5633     my $defaults = load_defaults();
5634
5635     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5636
5637     return PVE::Tools::cmd2string($cmd);
5638 }
5639
5640 sub vm_reset {
5641     my ($vmid, $skiplock) = @_;
5642
5643     PVE::QemuConfig->lock_config($vmid, sub {
5644
5645         my $conf = PVE::QemuConfig->load_config($vmid);
5646
5647         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5648
5649         vm_mon_cmd($vmid, "system_reset");
5650     });
5651 }
5652
5653 sub get_vm_volumes {
5654     my ($conf) = @_;
5655
5656     my $vollist = [];
5657     foreach_volid($conf, sub {
5658         my ($volid, $attr) = @_;
5659
5660         return if $volid =~ m|^/|;
5661
5662         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5663         return if !$sid;
5664
5665         push @$vollist, $volid;
5666     });
5667
5668     return $vollist;
5669 }
5670
5671 sub vm_stop_cleanup {
5672     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5673
5674     eval {
5675
5676         if (!$keepActive) {
5677             my $vollist = get_vm_volumes($conf);
5678             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5679         }
5680
5681         foreach my $ext (qw(mon qmp pid vnc qga)) {
5682             unlink "/var/run/qemu-server/${vmid}.$ext";
5683         }
5684
5685         if ($conf->{ivshmem}) {
5686             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5687             # just delete it for now, VMs which have this already open do not
5688             # are affected, but new VMs will get a separated one. If this
5689             # becomes an issue we either add some sort of ref-counting or just
5690             # add a "don't delete on stop" flag to the ivshmem format.
5691             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5692         }
5693
5694         foreach my $key (keys %$conf) {
5695             next if $key !~ m/^hostpci(\d+)$/;
5696             my $hostpciindex = $1;
5697             my $d = parse_hostpci($conf->{$key});
5698             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5699
5700             foreach my $pci (@{$d->{pciid}}) {
5701                 my $pciid = $pci->{id};
5702                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5703             }
5704         }
5705
5706         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5707     };
5708     warn $@ if $@; # avoid errors - just warn
5709 }
5710
5711 # call only in locked context
5712 sub _do_vm_stop {
5713     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5714
5715     my $pid = check_running($vmid, $nocheck);
5716     return if !$pid;
5717
5718     my $conf;
5719     if (!$nocheck) {
5720         $conf = PVE::QemuConfig->load_config($vmid);
5721         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5722         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5723             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5724             $timeout = $opts->{down} if $opts->{down};
5725         }
5726         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5727     }
5728
5729     eval {
5730         if ($shutdown) {
5731             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5732                 vm_qmp_command($vmid, {
5733                         execute => "guest-shutdown",
5734                         arguments => { timeout => $timeout }
5735                     }, $nocheck);
5736             } else {
5737                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5738             }
5739         } else {
5740             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5741         }
5742     };
5743     my $err = $@;
5744
5745     if (!$err) {
5746         $timeout = 60 if !defined($timeout);
5747
5748         my $count = 0;
5749         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5750             $count++;
5751             sleep 1;
5752         }
5753
5754         if ($count >= $timeout) {
5755             if ($force) {
5756                 warn "VM still running - terminating now with SIGTERM\n";
5757                 kill 15, $pid;
5758             } else {
5759                 die "VM quit/powerdown failed - got timeout\n";
5760             }
5761         } else {
5762             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5763             return;
5764         }
5765     } else {
5766         if ($force) {
5767             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5768             kill 15, $pid;
5769         } else {
5770             die "VM quit/powerdown failed\n";
5771         }
5772     }
5773
5774     # wait again
5775     $timeout = 10;
5776
5777     my $count = 0;
5778     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5779         $count++;
5780         sleep 1;
5781     }
5782
5783     if ($count >= $timeout) {
5784         warn "VM still running - terminating now with SIGKILL\n";
5785         kill 9, $pid;
5786         sleep 1;
5787     }
5788
5789     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5790 }
5791
5792 # Note: use $nocheck to skip tests if VM configuration file exists.
5793 # We need that when migration VMs to other nodes (files already moved)
5794 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5795 sub vm_stop {
5796     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5797
5798     $force = 1 if !defined($force) && !$shutdown;
5799
5800     if ($migratedfrom){
5801         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5802         kill 15, $pid if $pid;
5803         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5804         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5805         return;
5806     }
5807
5808     PVE::QemuConfig->lock_config($vmid, sub {
5809         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5810    });
5811 }
5812
5813 sub vm_reboot {
5814     my ($vmid, $timeout) = @_;
5815
5816     PVE::QemuConfig->lock_config($vmid, sub {
5817
5818         # only reboot if running, as qmeventd starts it again on a stop event
5819         return if !check_running($vmid);
5820
5821         create_reboot_request($vmid);
5822
5823         my $storecfg = PVE::Storage::config();
5824         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5825
5826    });
5827 }
5828
5829 sub vm_suspend {
5830     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5831
5832     my $conf;
5833     my $path;
5834     my $storecfg;
5835     my $vmstate;
5836
5837     PVE::QemuConfig->lock_config($vmid, sub {
5838
5839         $conf = PVE::QemuConfig->load_config($vmid);
5840
5841         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5842         PVE::QemuConfig->check_lock($conf)
5843             if !($skiplock || $is_backing_up);
5844
5845         die "cannot suspend to disk during backup\n"
5846             if $is_backing_up && $includestate;
5847
5848         if ($includestate) {
5849             $conf->{lock} = 'suspending';
5850             my $date = strftime("%Y-%m-%d", localtime(time()));
5851             $storecfg = PVE::Storage::config();
5852             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5853             $path = PVE::Storage::path($storecfg, $vmstate);
5854             PVE::QemuConfig->write_config($vmid, $conf);
5855         } else {
5856             vm_mon_cmd($vmid, "stop");
5857         }
5858     });
5859
5860     if ($includestate) {
5861         # save vm state
5862         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5863
5864         eval {
5865             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5866             for(;;) {
5867                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5868                 if (!$state->{status}) {
5869                     die "savevm not active\n";
5870                 } elsif ($state->{status} eq 'active') {
5871                     sleep(1);
5872                     next;
5873                 } elsif ($state->{status} eq 'completed') {
5874                     print "State saved, quitting\n";
5875                     last;
5876                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5877                     die "query-savevm failed with error '$state->{error}'\n"
5878                 } else {
5879                     die "query-savevm returned status '$state->{status}'\n";
5880                 }
5881             }
5882         };
5883         my $err = $@;
5884
5885         PVE::QemuConfig->lock_config($vmid, sub {
5886             $conf = PVE::QemuConfig->load_config($vmid);
5887             if ($err) {
5888                 # cleanup, but leave suspending lock, to indicate something went wrong
5889                 eval {
5890                     vm_mon_cmd($vmid, "savevm-end");
5891                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5892                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5893                     delete $conf->@{qw(vmstate runningmachine)};
5894                     PVE::QemuConfig->write_config($vmid, $conf);
5895                 };
5896                 warn $@ if $@;
5897                 die $err;
5898             }
5899
5900             die "lock changed unexpectedly\n"
5901                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5902
5903             vm_qmp_command($vmid, { execute => "quit" });
5904             $conf->{lock} = 'suspended';
5905             PVE::QemuConfig->write_config($vmid, $conf);
5906         });
5907     }
5908 }
5909
5910 sub vm_resume {
5911     my ($vmid, $skiplock, $nocheck) = @_;
5912
5913     PVE::QemuConfig->lock_config($vmid, sub {
5914         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5915         my $res = $vm_mon_cmd->($vmid, 'query-status');
5916         my $resume_cmd = 'cont';
5917
5918         if ($res->{status} && $res->{status} eq 'suspended') {
5919             $resume_cmd = 'system_wakeup';
5920         }
5921
5922         if (!$nocheck) {
5923
5924             my $conf = PVE::QemuConfig->load_config($vmid);
5925
5926             PVE::QemuConfig->check_lock($conf)
5927                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5928         }
5929
5930         $vm_mon_cmd->($vmid, $resume_cmd);
5931     });
5932 }
5933
5934 sub vm_sendkey {
5935     my ($vmid, $skiplock, $key) = @_;
5936
5937     PVE::QemuConfig->lock_config($vmid, sub {
5938
5939         my $conf = PVE::QemuConfig->load_config($vmid);
5940
5941         # there is no qmp command, so we use the human monitor command
5942         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5943         die $res if $res ne '';
5944     });
5945 }
5946
5947 sub vm_destroy {
5948     my ($storecfg, $vmid, $skiplock) = @_;
5949
5950     PVE::QemuConfig->lock_config($vmid, sub {
5951
5952         my $conf = PVE::QemuConfig->load_config($vmid);
5953
5954         if (!check_running($vmid)) {
5955             destroy_vm($storecfg, $vmid, undef, $skiplock);
5956         } else {
5957             die "VM $vmid is running - destroy failed\n";
5958         }
5959     });
5960 }
5961
5962 # vzdump restore implementaion
5963
5964 sub tar_archive_read_firstfile {
5965     my $archive = shift;
5966
5967     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5968
5969     # try to detect archive type first
5970     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5971         die "unable to open file '$archive'\n";
5972     my $firstfile = <$fh>;
5973     kill 15, $pid;
5974     close $fh;
5975
5976     die "ERROR: archive contaions no data\n" if !$firstfile;
5977     chomp $firstfile;
5978
5979     return $firstfile;
5980 }
5981
5982 sub tar_restore_cleanup {
5983     my ($storecfg, $statfile) = @_;
5984
5985     print STDERR "starting cleanup\n";
5986
5987     if (my $fd = IO::File->new($statfile, "r")) {
5988         while (defined(my $line = <$fd>)) {
5989             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5990                 my $volid = $2;
5991                 eval {
5992                     if ($volid =~ m|^/|) {
5993                         unlink $volid || die 'unlink failed\n';
5994                     } else {
5995                         PVE::Storage::vdisk_free($storecfg, $volid);
5996                     }
5997                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5998                 };
5999                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6000             } else {
6001                 print STDERR "unable to parse line in statfile - $line";
6002             }
6003         }
6004         $fd->close();
6005     }
6006 }
6007
6008 sub restore_archive {
6009     my ($archive, $vmid, $user, $opts) = @_;
6010
6011     my $format = $opts->{format};
6012     my $comp;
6013
6014     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6015         $format = 'tar' if !$format;
6016         $comp = 'gzip';
6017     } elsif ($archive =~ m/\.tar$/) {
6018         $format = 'tar' if !$format;
6019     } elsif ($archive =~ m/.tar.lzo$/) {
6020         $format = 'tar' if !$format;
6021         $comp = 'lzop';
6022     } elsif ($archive =~ m/\.vma$/) {
6023         $format = 'vma' if !$format;
6024     } elsif ($archive =~ m/\.vma\.gz$/) {
6025         $format = 'vma' if !$format;
6026         $comp = 'gzip';
6027     } elsif ($archive =~ m/\.vma\.lzo$/) {
6028         $format = 'vma' if !$format;
6029         $comp = 'lzop';
6030     } else {
6031         $format = 'vma' if !$format; # default
6032     }
6033
6034     # try to detect archive format
6035     if ($format eq 'tar') {
6036         return restore_tar_archive($archive, $vmid, $user, $opts);
6037     } else {
6038         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6039     }
6040 }
6041
6042 sub restore_update_config_line {
6043     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6044
6045     return if $line =~ m/^\#qmdump\#/;
6046     return if $line =~ m/^\#vzdump\#/;
6047     return if $line =~ m/^lock:/;
6048     return if $line =~ m/^unused\d+:/;
6049     return if $line =~ m/^parent:/;
6050
6051     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6052     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6053         # try to convert old 1.X settings
6054         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6055         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6056             my ($model, $macaddr) = split(/\=/, $devconfig);
6057             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6058             my $net = {
6059                 model => $model,
6060                 bridge => "vmbr$ind",
6061                 macaddr => $macaddr,
6062             };
6063             my $netstr = print_net($net);
6064
6065             print $outfd "net$cookie->{netcount}: $netstr\n";
6066             $cookie->{netcount}++;
6067         }
6068     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6069         my ($id, $netstr) = ($1, $2);
6070         my $net = parse_net($netstr);
6071         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6072         $netstr = print_net($net);
6073         print $outfd "$id: $netstr\n";
6074     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6075         my $virtdev = $1;
6076         my $value = $3;
6077         my $di = parse_drive($virtdev, $value);
6078         if (defined($di->{backup}) && !$di->{backup}) {
6079             print $outfd "#$line";
6080         } elsif ($map->{$virtdev}) {
6081             delete $di->{format}; # format can change on restore
6082             $di->{file} = $map->{$virtdev};
6083             $value = print_drive($vmid, $di);
6084             print $outfd "$virtdev: $value\n";
6085         } else {
6086             print $outfd $line;
6087         }
6088     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6089         my $vmgenid = $1;
6090         if ($vmgenid ne '0') {
6091             # always generate a new vmgenid if there was a valid one setup
6092             $vmgenid = generate_uuid();
6093         }
6094         print $outfd "vmgenid: $vmgenid\n";
6095     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6096         my ($uuid, $uuid_str);
6097         UUID::generate($uuid);
6098         UUID::unparse($uuid, $uuid_str);
6099         my $smbios1 = parse_smbios1($2);
6100         $smbios1->{uuid} = $uuid_str;
6101         print $outfd $1.print_smbios1($smbios1)."\n";
6102     } else {
6103         print $outfd $line;
6104     }
6105 }
6106
6107 sub scan_volids {
6108     my ($cfg, $vmid) = @_;
6109
6110     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6111
6112     my $volid_hash = {};
6113     foreach my $storeid (keys %$info) {
6114         foreach my $item (@{$info->{$storeid}}) {
6115             next if !($item->{volid} && $item->{size});
6116             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6117             $volid_hash->{$item->{volid}} = $item;
6118         }
6119     }
6120
6121     return $volid_hash;
6122 }
6123
6124 sub is_volume_in_use {
6125     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6126
6127     my $path = PVE::Storage::path($storecfg, $volid);
6128
6129     my $scan_config = sub {
6130         my ($cref, $snapname) = @_;
6131
6132         foreach my $key (keys %$cref) {
6133             my $value = $cref->{$key};
6134             if (is_valid_drivename($key)) {
6135                 next if $skip_drive && $key eq $skip_drive;
6136                 my $drive = parse_drive($key, $value);
6137                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6138                 return 1 if $volid eq $drive->{file};
6139                 if ($drive->{file} =~ m!^/!) {
6140                     return 1 if $drive->{file} eq $path;
6141                 } else {
6142                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6143                     next if !$storeid;
6144                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6145                     next if !$scfg;
6146                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6147                 }
6148             }
6149         }
6150
6151         return 0;
6152     };
6153
6154     return 1 if &$scan_config($conf);
6155
6156     undef $skip_drive;
6157
6158     foreach my $snapname (keys %{$conf->{snapshots}}) {
6159         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6160     }
6161
6162     return 0;
6163 }
6164
6165 sub update_disksize {
6166     my ($vmid, $conf, $volid_hash) = @_;
6167
6168     my $changes;
6169     my $prefix = "VM $vmid:";
6170
6171     # used and unused disks
6172     my $referenced = {};
6173
6174     # Note: it is allowed to define multiple storages with same path (alias), so
6175     # we need to check both 'volid' and real 'path' (two different volid can point
6176     # to the same path).
6177
6178     my $referencedpath = {};
6179
6180     # update size info
6181     foreach my $opt (keys %$conf) {
6182         if (is_valid_drivename($opt)) {
6183             my $drive = parse_drive($opt, $conf->{$opt});
6184             my $volid = $drive->{file};
6185             next if !$volid;
6186
6187             $referenced->{$volid} = 1;
6188             if ($volid_hash->{$volid} &&
6189                 (my $path = $volid_hash->{$volid}->{path})) {
6190                 $referencedpath->{$path} = 1;
6191             }
6192
6193             next if drive_is_cdrom($drive);
6194             next if !$volid_hash->{$volid};
6195
6196             $drive->{size} = $volid_hash->{$volid}->{size};
6197             my $new = print_drive($vmid, $drive);
6198             if ($new ne $conf->{$opt}) {
6199                 $changes = 1;
6200                 $conf->{$opt} = $new;
6201                 print "$prefix update disk '$opt' information.\n";
6202             }
6203         }
6204     }
6205
6206     # remove 'unusedX' entry if volume is used
6207     foreach my $opt (keys %$conf) {
6208         next if $opt !~ m/^unused\d+$/;
6209         my $volid = $conf->{$opt};
6210         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6211         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6212             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6213             $changes = 1;
6214             delete $conf->{$opt};
6215         }
6216
6217         $referenced->{$volid} = 1;
6218         $referencedpath->{$path} = 1 if $path;
6219     }
6220
6221     foreach my $volid (sort keys %$volid_hash) {
6222         next if $volid =~ m/vm-$vmid-state-/;
6223         next if $referenced->{$volid};
6224         my $path = $volid_hash->{$volid}->{path};
6225         next if !$path; # just to be sure
6226         next if $referencedpath->{$path};
6227         $changes = 1;
6228         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6229         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6230         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6231     }
6232
6233     return $changes;
6234 }
6235
6236 sub rescan {
6237     my ($vmid, $nolock, $dryrun) = @_;
6238
6239     my $cfg = PVE::Storage::config();
6240
6241     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6242     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6243     foreach my $stor (keys %{$cfg->{ids}}) {
6244         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6245     }
6246
6247     print "rescan volumes...\n";
6248     my $volid_hash = scan_volids($cfg, $vmid);
6249
6250     my $updatefn =  sub {
6251         my ($vmid) = @_;
6252
6253         my $conf = PVE::QemuConfig->load_config($vmid);
6254
6255         PVE::QemuConfig->check_lock($conf);
6256
6257         my $vm_volids = {};
6258         foreach my $volid (keys %$volid_hash) {
6259             my $info = $volid_hash->{$volid};
6260             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6261         }
6262
6263         my $changes = update_disksize($vmid, $conf, $vm_volids);
6264
6265         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6266     };
6267
6268     if (defined($vmid)) {
6269         if ($nolock) {
6270             &$updatefn($vmid);
6271         } else {
6272             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6273         }
6274     } else {
6275         my $vmlist = config_list();
6276         foreach my $vmid (keys %$vmlist) {
6277             if ($nolock) {
6278                 &$updatefn($vmid);
6279             } else {
6280                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6281             }
6282         }
6283     }
6284 }
6285
6286 sub restore_vma_archive {
6287     my ($archive, $vmid, $user, $opts, $comp) = @_;
6288
6289     my $readfrom = $archive;
6290
6291     my $cfg = PVE::Storage::config();
6292     my $commands = [];
6293     my $bwlimit = $opts->{bwlimit};
6294
6295     my $dbg_cmdstring = '';
6296     my $add_pipe = sub {
6297         my ($cmd) = @_;
6298         push @$commands, $cmd;
6299         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6300         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6301         $readfrom = '-';
6302     };
6303
6304     my $input = undef;
6305     if ($archive eq '-') {
6306         $input = '<&STDIN';
6307     } else {
6308         # If we use a backup from a PVE defined storage we also consider that
6309         # storage's rate limit:
6310         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6311         if (defined($volid)) {
6312             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6313             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6314             if ($readlimit) {
6315                 print STDERR "applying read rate limit: $readlimit\n";
6316                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6317                 $add_pipe->($cstream);
6318             }
6319         }
6320     }
6321
6322     if ($comp) {
6323         my $cmd;
6324         if ($comp eq 'gzip') {
6325             $cmd = ['zcat', $readfrom];
6326         } elsif ($comp eq 'lzop') {
6327             $cmd = ['lzop', '-d', '-c', $readfrom];
6328         } else {
6329             die "unknown compression method '$comp'\n";
6330         }
6331         $add_pipe->($cmd);
6332     }
6333
6334     my $tmpdir = "/var/tmp/vzdumptmp$$";
6335     rmtree $tmpdir;
6336
6337     # disable interrupts (always do cleanups)
6338     local $SIG{INT} =
6339         local $SIG{TERM} =
6340         local $SIG{QUIT} =
6341         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6342
6343     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6344     POSIX::mkfifo($mapfifo, 0600);
6345     my $fifofh;
6346
6347     my $openfifo = sub {
6348         open($fifofh, '>', $mapfifo) || die $!;
6349     };
6350
6351     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6352
6353     my $oldtimeout;
6354     my $timeout = 5;
6355
6356     my $devinfo = {};
6357
6358     my $rpcenv = PVE::RPCEnvironment::get();
6359
6360     my $conffile = PVE::QemuConfig->config_file($vmid);
6361     my $tmpfn = "$conffile.$$.tmp";
6362
6363     # Note: $oldconf is undef if VM does not exists
6364     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6365     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6366
6367     my %storage_limits;
6368
6369     my $print_devmap = sub {
6370         my $virtdev_hash = {};
6371
6372         my $cfgfn = "$tmpdir/qemu-server.conf";
6373
6374         # we can read the config - that is already extracted
6375         my $fh = IO::File->new($cfgfn, "r") ||
6376             "unable to read qemu-server.conf - $!\n";
6377
6378         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6379         if (-f $fwcfgfn) {
6380             my $pve_firewall_dir = '/etc/pve/firewall';
6381             mkdir $pve_firewall_dir; # make sure the dir exists
6382             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6383         }
6384
6385         while (defined(my $line = <$fh>)) {
6386             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6387                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6388                 die "archive does not contain data for drive '$virtdev'\n"
6389                     if !$devinfo->{$devname};
6390                 if (defined($opts->{storage})) {
6391                     $storeid = $opts->{storage} || 'local';
6392                 } elsif (!$storeid) {
6393                     $storeid = 'local';
6394                 }
6395                 $format = 'raw' if !$format;
6396                 $devinfo->{$devname}->{devname} = $devname;
6397                 $devinfo->{$devname}->{virtdev} = $virtdev;
6398                 $devinfo->{$devname}->{format} = $format;
6399                 $devinfo->{$devname}->{storeid} = $storeid;
6400
6401                 # check permission on storage
6402                 my $pool = $opts->{pool}; # todo: do we need that?
6403                 if ($user ne 'root@pam') {
6404                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6405                 }
6406
6407                 $storage_limits{$storeid} = $bwlimit;
6408
6409                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6410             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6411                 my $virtdev = $1;
6412                 my $drive = parse_drive($virtdev, $2);
6413                 if (drive_is_cloudinit($drive)) {
6414                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6415                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6416                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6417
6418                     my $d = {
6419                         format => $format,
6420                         storeid => $opts->{storage} // $storeid,
6421                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6422                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6423                         name => "vm-$vmid-cloudinit",
6424                         is_cloudinit => 1,
6425                     };
6426                     $virtdev_hash->{$virtdev} = $d;
6427                 }
6428             }
6429         }
6430
6431         foreach my $key (keys %storage_limits) {
6432             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6433             next if !$limit;
6434             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6435             $storage_limits{$key} = $limit * 1024;
6436         }
6437
6438         foreach my $devname (keys %$devinfo) {
6439             die "found no device mapping information for device '$devname'\n"
6440                 if !$devinfo->{$devname}->{virtdev};
6441         }
6442
6443         # create empty/temp config
6444         if ($oldconf) {
6445             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6446             foreach_drive($oldconf, sub {
6447                 my ($ds, $drive) = @_;
6448
6449                 return if drive_is_cdrom($drive, 1);
6450
6451                 my $volid = $drive->{file};
6452                 return if !$volid || $volid =~ m|^/|;
6453
6454                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6455                 return if !$path || !$owner || ($owner != $vmid);
6456
6457                 # Note: only delete disk we want to restore
6458                 # other volumes will become unused
6459                 if ($virtdev_hash->{$ds}) {
6460                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6461                     if (my $err = $@) {
6462                         warn $err;
6463                     }
6464                 }
6465             });
6466
6467             # delete vmstate files, after the restore we have no snapshots anymore
6468             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6469                 my $snap = $oldconf->{snapshots}->{$snapname};
6470                 if ($snap->{vmstate}) {
6471                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6472                     if (my $err = $@) {
6473                         warn $err;
6474                     }
6475                 }
6476             }
6477         }
6478
6479         my $map = {};
6480         foreach my $virtdev (sort keys %$virtdev_hash) {
6481             my $d = $virtdev_hash->{$virtdev};
6482             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6483             my $storeid = $d->{storeid};
6484             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6485
6486             my $map_opts = '';
6487             if (my $limit = $storage_limits{$storeid}) {
6488                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6489             }
6490
6491             # test if requested format is supported
6492             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6493             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6494             $d->{format} = $defFormat if !$supported;
6495
6496             my $name;
6497             if ($d->{is_cloudinit}) {
6498                 $name = $d->{name};
6499                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6500             }
6501
6502             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6503             print STDERR "new volume ID is '$volid'\n";
6504             $d->{volid} = $volid;
6505
6506             PVE::Storage::activate_volumes($cfg, [$volid]);
6507
6508             my $write_zeros = 1;
6509             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6510                 $write_zeros = 0;
6511             }
6512
6513             if (!$d->{is_cloudinit}) {
6514                 my $path = PVE::Storage::path($cfg, $volid);
6515
6516                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6517
6518                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6519             }
6520             $map->{$virtdev} = $volid;
6521         }
6522
6523         $fh->seek(0, 0) || die "seek failed - $!\n";
6524
6525         my $outfd = new IO::File ($tmpfn, "w") ||
6526             die "unable to write config for VM $vmid\n";
6527
6528         my $cookie = { netcount => 0 };
6529         while (defined(my $line = <$fh>)) {
6530             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6531         }
6532
6533         $fh->close();
6534         $outfd->close();
6535     };
6536
6537     eval {
6538         # enable interrupts
6539         local $SIG{INT} =
6540             local $SIG{TERM} =
6541             local $SIG{QUIT} =
6542             local $SIG{HUP} =
6543             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6544         local $SIG{ALRM} = sub { die "got timeout\n"; };
6545
6546         $oldtimeout = alarm($timeout);
6547
6548         my $parser = sub {
6549             my $line = shift;
6550
6551             print "$line\n";
6552
6553             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6554                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6555                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6556             } elsif ($line =~ m/^CTIME: /) {
6557                 # we correctly received the vma config, so we can disable
6558                 # the timeout now for disk allocation (set to 10 minutes, so
6559                 # that we always timeout if something goes wrong)
6560                 alarm(600);
6561                 &$print_devmap();
6562                 print $fifofh "done\n";
6563                 my $tmp = $oldtimeout || 0;
6564                 $oldtimeout = undef;
6565                 alarm($tmp);
6566                 close($fifofh);
6567             }
6568         };
6569
6570         print "restore vma archive: $dbg_cmdstring\n";
6571         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6572     };
6573     my $err = $@;
6574
6575     alarm($oldtimeout) if $oldtimeout;
6576
6577     my $vollist = [];
6578     foreach my $devname (keys %$devinfo) {
6579         my $volid = $devinfo->{$devname}->{volid};
6580         push @$vollist, $volid if $volid;
6581     }
6582
6583     PVE::Storage::deactivate_volumes($cfg, $vollist);
6584
6585     unlink $mapfifo;
6586
6587     if ($err) {
6588         rmtree $tmpdir;
6589         unlink $tmpfn;
6590
6591         foreach my $devname (keys %$devinfo) {
6592             my $volid = $devinfo->{$devname}->{volid};
6593             next if !$volid;
6594             eval {
6595                 if ($volid =~ m|^/|) {
6596                     unlink $volid || die 'unlink failed\n';
6597                 } else {
6598                     PVE::Storage::vdisk_free($cfg, $volid);
6599                 }
6600                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6601             };
6602             print STDERR "unable to cleanup '$volid' - $@" if $@;
6603         }
6604         die $err;
6605     }
6606
6607     rmtree $tmpdir;
6608
6609     rename($tmpfn, $conffile) ||
6610         die "unable to commit configuration file '$conffile'\n";
6611
6612     PVE::Cluster::cfs_update(); # make sure we read new file
6613
6614     eval { rescan($vmid, 1); };
6615     warn $@ if $@;
6616 }
6617
6618 sub restore_tar_archive {
6619     my ($archive, $vmid, $user, $opts) = @_;
6620
6621     if ($archive ne '-') {
6622         my $firstfile = tar_archive_read_firstfile($archive);
6623         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6624             if $firstfile ne 'qemu-server.conf';
6625     }
6626
6627     my $storecfg = PVE::Storage::config();
6628
6629     # destroy existing data - keep empty config
6630     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6631     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6632
6633     my $tocmd = "/usr/lib/qemu-server/qmextract";
6634
6635     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6636     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6637     $tocmd .= ' --prealloc' if $opts->{prealloc};
6638     $tocmd .= ' --info' if $opts->{info};
6639
6640     # tar option "xf" does not autodetect compression when read from STDIN,
6641     # so we pipe to zcat
6642     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6643         PVE::Tools::shellquote("--to-command=$tocmd");
6644
6645     my $tmpdir = "/var/tmp/vzdumptmp$$";
6646     mkpath $tmpdir;
6647
6648     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6649     local $ENV{VZDUMP_VMID} = $vmid;
6650     local $ENV{VZDUMP_USER} = $user;
6651
6652     my $conffile = PVE::QemuConfig->config_file($vmid);
6653     my $tmpfn = "$conffile.$$.tmp";
6654
6655     # disable interrupts (always do cleanups)
6656     local $SIG{INT} =
6657         local $SIG{TERM} =
6658         local $SIG{QUIT} =
6659         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6660
6661     eval {
6662         # enable interrupts
6663         local $SIG{INT} =
6664             local $SIG{TERM} =
6665             local $SIG{QUIT} =
6666             local $SIG{HUP} =
6667             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6668
6669         if ($archive eq '-') {
6670             print "extracting archive from STDIN\n";
6671             run_command($cmd, input => "<&STDIN");
6672         } else {
6673             print "extracting archive '$archive'\n";
6674             run_command($cmd);
6675         }
6676
6677         return if $opts->{info};
6678
6679         # read new mapping
6680         my $map = {};
6681         my $statfile = "$tmpdir/qmrestore.stat";
6682         if (my $fd = IO::File->new($statfile, "r")) {
6683             while (defined (my $line = <$fd>)) {
6684                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6685                     $map->{$1} = $2 if $1;
6686                 } else {
6687                     print STDERR "unable to parse line in statfile - $line\n";
6688                 }
6689             }
6690             $fd->close();
6691         }
6692
6693         my $confsrc = "$tmpdir/qemu-server.conf";
6694
6695         my $srcfd = new IO::File($confsrc, "r") ||
6696             die "unable to open file '$confsrc'\n";
6697
6698         my $outfd = new IO::File ($tmpfn, "w") ||
6699             die "unable to write config for VM $vmid\n";
6700
6701         my $cookie = { netcount => 0 };
6702         while (defined (my $line = <$srcfd>)) {
6703             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6704         }
6705
6706         $srcfd->close();
6707         $outfd->close();
6708     };
6709     my $err = $@;
6710
6711     if ($err) {
6712
6713         unlink $tmpfn;
6714
6715         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6716
6717         die $err;
6718     }
6719
6720     rmtree $tmpdir;
6721
6722     rename $tmpfn, $conffile ||
6723         die "unable to commit configuration file '$conffile'\n";
6724
6725     PVE::Cluster::cfs_update(); # make sure we read new file
6726
6727     eval { rescan($vmid, 1); };
6728     warn $@ if $@;
6729 };
6730
6731 sub foreach_storage_used_by_vm {
6732     my ($conf, $func) = @_;
6733
6734     my $sidhash = {};
6735
6736     foreach_drive($conf, sub {
6737         my ($ds, $drive) = @_;
6738         return if drive_is_cdrom($drive);
6739
6740         my $volid = $drive->{file};
6741
6742         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6743         $sidhash->{$sid} = $sid if $sid;
6744     });
6745
6746     foreach my $sid (sort keys %$sidhash) {
6747         &$func($sid);
6748     }
6749 }
6750
6751 sub do_snapshots_with_qemu {
6752     my ($storecfg, $volid) = @_;
6753
6754     my $storage_name = PVE::Storage::parse_volume_id($volid);
6755     my $scfg = $storecfg->{ids}->{$storage_name};
6756
6757     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6758         return 1;
6759     }
6760
6761     if ($volid =~ m/\.(qcow2|qed)$/){
6762         return 1;
6763     }
6764
6765     return undef;
6766 }
6767
6768 sub qga_check_running {
6769     my ($vmid, $nowarn) = @_;
6770
6771     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6772     if ($@) {
6773         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6774         return 0;
6775     }
6776     return 1;
6777 }
6778
6779 sub template_create {
6780     my ($vmid, $conf, $disk) = @_;
6781
6782     my $storecfg = PVE::Storage::config();
6783
6784     foreach_drive($conf, sub {
6785         my ($ds, $drive) = @_;
6786
6787         return if drive_is_cdrom($drive);
6788         return if $disk && $ds ne $disk;
6789
6790         my $volid = $drive->{file};
6791         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6792
6793         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6794         $drive->{file} = $voliddst;
6795         $conf->{$ds} = print_drive($vmid, $drive);
6796         PVE::QemuConfig->write_config($vmid, $conf);
6797     });
6798 }
6799
6800 sub convert_iscsi_path {
6801     my ($path) = @_;
6802
6803     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6804         my $portal = $1;
6805         my $target = $2;
6806         my $lun = $3;
6807
6808         my $initiator_name = get_initiator_name();
6809
6810         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6811                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6812     }
6813
6814     die "cannot convert iscsi path '$path', unkown format\n";
6815 }
6816
6817 sub qemu_img_convert {
6818     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6819
6820     my $storecfg = PVE::Storage::config();
6821     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6822     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6823
6824     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6825
6826     my $cachemode;
6827     my $src_path;
6828     my $src_is_iscsi = 0;
6829     my $src_format = 'raw';
6830
6831     if ($src_storeid) {
6832         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6833         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6834         $src_format = qemu_img_format($src_scfg, $src_volname);
6835         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6836         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6837         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6838     } elsif (-f $src_volid) {
6839         $src_path = $src_volid;
6840         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6841             $src_format = $1;
6842         }
6843     }
6844
6845     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6846
6847     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6848     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6849     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6850     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6851
6852     my $cmd = [];
6853     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6854     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6855     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6856     push @$cmd, '-T', $cachemode if defined($cachemode);
6857
6858     if ($src_is_iscsi) {
6859         push @$cmd, '--image-opts';
6860         $src_path = convert_iscsi_path($src_path);
6861     } else {
6862         push @$cmd, '-f', $src_format;
6863     }
6864
6865     if ($dst_is_iscsi) {
6866         push @$cmd, '--target-image-opts';
6867         $dst_path = convert_iscsi_path($dst_path);
6868     } else {
6869         push @$cmd, '-O', $dst_format;
6870     }
6871
6872     push @$cmd, $src_path;
6873
6874     if (!$dst_is_iscsi && $is_zero_initialized) {
6875         push @$cmd, "zeroinit:$dst_path";
6876     } else {
6877         push @$cmd, $dst_path;
6878     }
6879
6880     my $parser = sub {
6881         my $line = shift;
6882         if($line =~ m/\((\S+)\/100\%\)/){
6883             my $percent = $1;
6884             my $transferred = int($size * $percent / 100);
6885             my $remaining = $size - $transferred;
6886
6887             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6888         }
6889
6890     };
6891
6892     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6893     my $err = $@;
6894     die "copy failed: $err" if $err;
6895 }
6896
6897 sub qemu_img_format {
6898     my ($scfg, $volname) = @_;
6899
6900     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6901         return $1;
6902     } else {
6903         return "raw";
6904     }
6905 }
6906
6907 sub qemu_drive_mirror {
6908     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6909
6910     $jobs = {} if !$jobs;
6911
6912     my $qemu_target;
6913     my $format;
6914     $jobs->{"drive-$drive"} = {};
6915
6916     if ($dst_volid =~ /^nbd:/) {
6917         $qemu_target = $dst_volid;
6918         $format = "nbd";
6919     } else {
6920         my $storecfg = PVE::Storage::config();
6921         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6922
6923         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6924
6925         $format = qemu_img_format($dst_scfg, $dst_volname);
6926
6927         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6928
6929         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6930     }
6931
6932     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6933     $opts->{format} = $format if $format;
6934
6935     if (defined($bwlimit)) {
6936         $opts->{speed} = $bwlimit * 1024;
6937         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6938     } else {
6939         print "drive mirror is starting for drive-$drive\n";
6940     }
6941
6942     # if a job already runs for this device we get an error, catch it for cleanup
6943     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6944     if (my $err = $@) {
6945         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6946         warn "$@\n" if $@;
6947         die "mirroring error: $err\n";
6948     }
6949
6950     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6951 }
6952
6953 sub qemu_drive_mirror_monitor {
6954     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6955
6956     eval {
6957         my $err_complete = 0;
6958
6959         while (1) {
6960             die "storage migration timed out\n" if $err_complete > 300;
6961
6962             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6963
6964             my $running_mirror_jobs = {};
6965             foreach my $stat (@$stats) {
6966                 next if $stat->{type} ne 'mirror';
6967                 $running_mirror_jobs->{$stat->{device}} = $stat;
6968             }
6969
6970             my $readycounter = 0;
6971
6972             foreach my $job (keys %$jobs) {
6973
6974                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6975                     print "$job : finished\n";
6976                     delete $jobs->{$job};
6977                     next;
6978                 }
6979
6980                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6981
6982                 my $busy = $running_mirror_jobs->{$job}->{busy};
6983                 my $ready = $running_mirror_jobs->{$job}->{ready};
6984                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6985                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6986                     my $remaining = $total - $transferred;
6987                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6988
6989                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6990                 }
6991
6992                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6993             }
6994
6995             last if scalar(keys %$jobs) == 0;
6996
6997             if ($readycounter == scalar(keys %$jobs)) {
6998                 print "all mirroring jobs are ready \n";
6999                 last if $skipcomplete; #do the complete later
7000
7001                 if ($vmiddst && $vmiddst != $vmid) {
7002                     my $agent_running = $qga && qga_check_running($vmid);
7003                     if ($agent_running) {
7004                         print "freeze filesystem\n";
7005                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7006                     } else {
7007                         print "suspend vm\n";
7008                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7009                     }
7010
7011                     # if we clone a disk for a new target vm, we don't switch the disk
7012                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7013
7014                     if ($agent_running) {
7015                         print "unfreeze filesystem\n";
7016                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7017                     } else {
7018                         print "resume vm\n";
7019                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7020                     }
7021
7022                     last;
7023                 } else {
7024
7025                     foreach my $job (keys %$jobs) {
7026                         # try to switch the disk if source and destination are on the same guest
7027                         print "$job: Completing block job...\n";
7028
7029                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7030                         if ($@ =~ m/cannot be completed/) {
7031                             print "$job: Block job cannot be completed, try again.\n";
7032                             $err_complete++;
7033                         }else {
7034                             print "$job: Completed successfully.\n";
7035                             $jobs->{$job}->{complete} = 1;
7036                         }
7037                     }
7038                 }
7039             }
7040             sleep 1;
7041         }
7042     };
7043     my $err = $@;
7044
7045     if ($err) {
7046         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7047         die "mirroring error: $err";
7048     }
7049
7050 }
7051
7052 sub qemu_blockjobs_cancel {
7053     my ($vmid, $jobs) = @_;
7054
7055     foreach my $job (keys %$jobs) {
7056         print "$job: Cancelling block job\n";
7057         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7058         $jobs->{$job}->{cancel} = 1;
7059     }
7060
7061     while (1) {
7062         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7063
7064         my $running_jobs = {};
7065         foreach my $stat (@$stats) {
7066             $running_jobs->{$stat->{device}} = $stat;
7067         }
7068
7069         foreach my $job (keys %$jobs) {
7070
7071             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7072                 print "$job: Done.\n";
7073                 delete $jobs->{$job};
7074             }
7075         }
7076
7077         last if scalar(keys %$jobs) == 0;
7078
7079         sleep 1;
7080     }
7081 }
7082
7083 sub clone_disk {
7084     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7085         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7086
7087     my $newvolid;
7088
7089     if (!$full) {
7090         print "create linked clone of drive $drivename ($drive->{file})\n";
7091         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7092         push @$newvollist, $newvolid;
7093     } else {
7094
7095         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7096         $storeid = $storage if $storage;
7097
7098         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7099         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7100
7101         print "create full clone of drive $drivename ($drive->{file})\n";
7102         my $name = undef;
7103         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7104         push @$newvollist, $newvolid;
7105
7106         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7107
7108         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7109         if (!$running || $snapname) {
7110             # TODO: handle bwlimits
7111             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7112         } else {
7113
7114             my $kvmver = get_running_qemu_version ($vmid);
7115             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7116                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7117                     if $drive->{iothread};
7118             }
7119
7120             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7121         }
7122     }
7123
7124     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7125
7126     my $disk = $drive;
7127     $disk->{format} = undef;
7128     $disk->{file} = $newvolid;
7129     $disk->{size} = $size;
7130
7131     return $disk;
7132 }
7133
7134 # this only works if VM is running
7135 sub get_current_qemu_machine {
7136     my ($vmid) = @_;
7137
7138     my $cmd = { execute => 'query-machines', arguments => {} };
7139     my $res = vm_qmp_command($vmid, $cmd);
7140
7141     my ($current, $default);
7142     foreach my $e (@$res) {
7143         $default = $e->{name} if $e->{'is-default'};
7144         $current = $e->{name} if $e->{'is-current'};
7145     }
7146
7147     # fallback to the default machine if current is not supported by qemu
7148     return $current || $default || 'pc';
7149 }
7150
7151 sub get_running_qemu_version {
7152     my ($vmid) = @_;
7153     my $cmd = { execute => 'query-version', arguments => {} };
7154     my $res = vm_qmp_command($vmid, $cmd);
7155     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7156 }
7157
7158 sub qemu_machine_feature_enabled {
7159     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7160
7161     my $current_major;
7162     my $current_minor;
7163
7164     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7165
7166         $current_major = $3;
7167         $current_minor = $4;
7168
7169     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7170
7171         $current_major = $1;
7172         $current_minor = $2;
7173     }
7174
7175     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7176 }
7177
7178 # gets in pairs the versions you want to compares, i.e.:
7179 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7180 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7181 sub version_cmp {
7182     my @versions = @_;
7183
7184     my $size = scalar(@versions);
7185
7186     return 0 if $size == 0;
7187     die "cannot compare odd count of versions" if $size & 1;
7188
7189     for (my $i = 0; $i < $size; $i += 2) {
7190         my ($a, $b) = splice(@versions, 0, 2);
7191         $a //= 0;
7192         $b //= 0;
7193
7194         return 1 if $a > $b;
7195         return -1 if $a < $b;
7196     }
7197     return 0;
7198 }
7199
7200 sub runs_at_least_qemu_version {
7201     my ($vmid, $major, $minor, $extra) = @_;
7202
7203     my $v = eval { PVE::QemuServer::vm_qmp_command($vmid, { execute => 'query-version' }) } // {};
7204     $v = $v->{qemu};
7205
7206     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7207 }
7208
7209 sub qemu_machine_pxe {
7210     my ($vmid, $conf) = @_;
7211
7212     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7213
7214     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7215         $machine .= '.pxe';
7216     }
7217
7218     return $machine;
7219 }
7220
7221 sub qemu_use_old_bios_files {
7222     my ($machine_type) = @_;
7223
7224     return if !$machine_type;
7225
7226     my $use_old_bios_files = undef;
7227
7228     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7229         $machine_type = $1;
7230         $use_old_bios_files = 1;
7231     } else {
7232         my $kvmver = kvm_user_version();
7233         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7234         # load new efi bios files on migration. So this hack is required to allow
7235         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7236         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7237         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7238     }
7239
7240     return ($use_old_bios_files, $machine_type);
7241 }
7242
7243 sub create_efidisk($$$$$) {
7244     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7245
7246     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7247     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7248
7249     my $vars_size_b = -s $ovmf_vars;
7250     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7251     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7252     PVE::Storage::activate_volumes($storecfg, [$volid]);
7253
7254     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7255
7256     return ($volid, $vars_size);
7257 }
7258
7259 sub vm_iothreads_list {
7260     my ($vmid) = @_;
7261
7262     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7263
7264     my $iothreads = {};
7265     foreach my $iothread (@$res) {
7266         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7267     }
7268
7269     return $iothreads;
7270 }
7271
7272 sub scsihw_infos {
7273     my ($conf, $drive) = @_;
7274
7275     my $maxdev = 0;
7276
7277     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7278         $maxdev = 7;
7279     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7280         $maxdev = 1;
7281     } else {
7282         $maxdev = 256;
7283     }
7284
7285     my $controller = int($drive->{index} / $maxdev);
7286     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7287
7288     return ($maxdev, $controller, $controller_prefix);
7289 }
7290
7291 sub add_hyperv_enlightenments {
7292     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7293
7294     return if $winversion < 6;
7295     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7296
7297     if ($gpu_passthrough || defined($hv_vendor_id)) {
7298         $hv_vendor_id //= 'proxmox';
7299         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7300     }
7301
7302     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7303         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7304         push @$cpuFlags , 'hv_vapic';
7305         push @$cpuFlags , 'hv_time';
7306     } else {
7307         push @$cpuFlags , 'hv_spinlocks=0xffff';
7308     }
7309
7310     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7311         push @$cpuFlags , 'hv_reset';
7312         push @$cpuFlags , 'hv_vpindex';
7313         push @$cpuFlags , 'hv_runtime';
7314     }
7315
7316     if ($winversion >= 7) {
7317         push @$cpuFlags , 'hv_relaxed';
7318
7319         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7320             push @$cpuFlags , 'hv_synic';
7321             push @$cpuFlags , 'hv_stimer';
7322         }
7323
7324         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7325             push @$cpuFlags , 'hv_ipi';
7326         }
7327     }
7328 }
7329
7330 sub windows_version {
7331     my ($ostype) = @_;
7332
7333     return 0 if !$ostype;
7334
7335     my $winversion = 0;
7336
7337     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7338         $winversion = 5;
7339     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7340         $winversion = 6;
7341     } elsif ($ostype =~ m/^win(\d+)$/) {
7342         $winversion = $1;
7343     }
7344
7345     return $winversion;
7346 }
7347
7348 sub resolve_dst_disk_format {
7349         my ($storecfg, $storeid, $src_volname, $format) = @_;
7350         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7351
7352         if (!$format) {
7353             # if no target format is specified, use the source disk format as hint
7354             if ($src_volname) {
7355                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7356                 $format = qemu_img_format($scfg, $src_volname);
7357             } else {
7358                 return $defFormat;
7359             }
7360         }
7361
7362         # test if requested format is supported - else use default
7363         my $supported = grep { $_ eq $format } @$validFormats;
7364         $format = $defFormat if !$supported;
7365         return $format;
7366 }
7367
7368 sub resolve_first_disk {
7369     my $conf = shift;
7370     my @disks = PVE::QemuServer::valid_drive_names();
7371     my $firstdisk;
7372     foreach my $ds (reverse @disks) {
7373         next if !$conf->{$ds};
7374         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7375         next if PVE::QemuServer::drive_is_cdrom($disk);
7376         $firstdisk = $ds;
7377     }
7378     return $firstdisk;
7379 }
7380
7381 sub generate_uuid {
7382     my ($uuid, $uuid_str);
7383     UUID::generate($uuid);
7384     UUID::unparse($uuid, $uuid_str);
7385     return $uuid_str;
7386 }
7387
7388 sub generate_smbios1_uuid {
7389     return "uuid=".generate_uuid();
7390 }
7391
7392 sub nbd_stop {
7393     my ($vmid) = @_;
7394
7395     vm_mon_cmd($vmid, 'nbd-server-stop');
7396 }
7397
7398 sub create_reboot_request {
7399     my ($vmid) = @_;
7400     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7401         or die "failed to create reboot trigger file: $!\n";
7402     close($fh);
7403 }
7404
7405 sub clear_reboot_request {
7406     my ($vmid) = @_;
7407     my $path = "/run/qemu-server/$vmid.reboot";
7408     my $res = 0;
7409
7410     $res = unlink($path);
7411     die "could not remove reboot request for $vmid: $!"
7412         if !$res && $! != POSIX::ENOENT;
7413
7414     return $res;
7415 }
7416
7417 # bash completion helper
7418
7419 sub complete_backup_archives {
7420     my ($cmdname, $pname, $cvalue) = @_;
7421
7422     my $cfg = PVE::Storage::config();
7423
7424     my $storeid;
7425
7426     if ($cvalue =~ m/^([^:]+):/) {
7427         $storeid = $1;
7428     }
7429
7430     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7431
7432     my $res = [];
7433     foreach my $id (keys %$data) {
7434         foreach my $item (@{$data->{$id}}) {
7435             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7436             push @$res, $item->{volid} if defined($item->{volid});
7437         }
7438     }
7439
7440     return $res;
7441 }
7442
7443 my $complete_vmid_full = sub {
7444     my ($running) = @_;
7445
7446     my $idlist = vmstatus();
7447
7448     my $res = [];
7449
7450     foreach my $id (keys %$idlist) {
7451         my $d = $idlist->{$id};
7452         if (defined($running)) {
7453             next if $d->{template};
7454             next if $running && $d->{status} ne 'running';
7455             next if !$running && $d->{status} eq 'running';
7456         }
7457         push @$res, $id;
7458
7459     }
7460     return $res;
7461 };
7462
7463 sub complete_vmid {
7464     return &$complete_vmid_full();
7465 }
7466
7467 sub complete_vmid_stopped {
7468     return &$complete_vmid_full(0);
7469 }
7470
7471 sub complete_vmid_running {
7472     return &$complete_vmid_full(1);
7473 }
7474
7475 sub complete_storage {
7476
7477     my $cfg = PVE::Storage::config();
7478     my $ids = $cfg->{ids};
7479
7480     my $res = [];
7481     foreach my $sid (keys %$ids) {
7482         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7483         next if !$ids->{$sid}->{content}->{images};
7484         push @$res, $sid;
7485     }
7486
7487     return $res;
7488 }
7489
7490 1;