PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1902             if ($path =~ m/^iscsi\:\/\// &&
1903                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1904                 $devicetype = 'generic';
1905             }
1906         }
1907
1908         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         } else {
1911             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1912         }
1913
1914         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1915             $device .= ",rotation_rate=1";
1916         }
1917         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1918
1919     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1920         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1921         my $controller = int($drive->{index} / $maxdev);
1922         my $unit = $drive->{index} % $maxdev;
1923         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1924
1925         $device = "ide-$devicetype";
1926         if ($drive->{interface} eq 'ide') {
1927             $device .= ",bus=ide.$controller,unit=$unit";
1928         } else {
1929             $device .= ",bus=ahci$controller.$unit";
1930         }
1931         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1932
1933         if ($devicetype eq 'hd') {
1934             if (my $model = $drive->{model}) {
1935                 $model = URI::Escape::uri_unescape($model);
1936                 $device .= ",model=$model";
1937             }
1938             if ($drive->{ssd}) {
1939                 $device .= ",rotation_rate=1";
1940             }
1941         }
1942         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1943     } elsif ($drive->{interface} eq 'usb') {
1944         die "implement me";
1945         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1946     } else {
1947         die "unsupported interface type";
1948     }
1949
1950     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1951
1952     if (my $serial = $drive->{serial}) {
1953         $serial = URI::Escape::uri_unescape($serial);
1954         $device .= ",serial=$serial";
1955     }
1956
1957
1958     return $device;
1959 }
1960
1961 sub get_initiator_name {
1962     my $initiator;
1963
1964     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1965     while (defined(my $line = <$fh>)) {
1966         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1967         $initiator = $1;
1968         last;
1969     }
1970     $fh->close();
1971
1972     return $initiator;
1973 }
1974
1975 sub print_drive_full {
1976     my ($storecfg, $vmid, $drive) = @_;
1977
1978     my $path;
1979     my $volid = $drive->{file};
1980     my $format;
1981
1982     if (drive_is_cdrom($drive)) {
1983         $path = get_iso_path($storecfg, $vmid, $volid);
1984     } else {
1985         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1986         if ($storeid) {
1987             $path = PVE::Storage::path($storecfg, $volid);
1988             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1989             $format = qemu_img_format($scfg, $volname);
1990         } else {
1991             $path = $volid;
1992             $format = "raw";
1993         }
1994    }
1995
1996     my $opts = '';
1997     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1998     foreach my $o (@qemu_drive_options) {
1999         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2000     }
2001
2002     # snapshot only accepts on|off
2003     if (defined($drive->{snapshot})) {
2004         my $v = $drive->{snapshot} ? 'on' : 'off';
2005         $opts .= ",snapshot=$v";
2006     }
2007
2008     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2009         my ($dir, $qmpname) = @$type;
2010         if (my $v = $drive->{"mbps$dir"}) {
2011             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2012         }
2013         if (my $v = $drive->{"mbps${dir}_max"}) {
2014             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2015         }
2016         if (my $v = $drive->{"bps${dir}_max_length"}) {
2017             $opts .= ",throttling.bps$qmpname-max-length=$v";
2018         }
2019         if (my $v = $drive->{"iops${dir}"}) {
2020             $opts .= ",throttling.iops$qmpname=$v";
2021         }
2022         if (my $v = $drive->{"iops${dir}_max"}) {
2023             $opts .= ",throttling.iops$qmpname-max=$v";
2024         }
2025         if (my $v = $drive->{"iops${dir}_max_length"}) {
2026             $opts .= ",throttling.iops$qmpname-max-length=$v";
2027         }
2028     }
2029
2030     $opts .= ",format=$format" if $format && !$drive->{format};
2031
2032     my $cache_direct = 0;
2033
2034     if (my $cache = $drive->{cache}) {
2035         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2036     } elsif (!drive_is_cdrom($drive)) {
2037         $opts .= ",cache=none";
2038         $cache_direct = 1;
2039     }
2040
2041     # aio native works only with O_DIRECT
2042     if (!$drive->{aio}) {
2043         if($cache_direct) {
2044             $opts .= ",aio=native";
2045         } else {
2046             $opts .= ",aio=threads";
2047         }
2048     }
2049
2050     if (!drive_is_cdrom($drive)) {
2051         my $detectzeroes;
2052         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2053             $detectzeroes = 'off';
2054         } elsif ($drive->{discard}) {
2055             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2056         } else {
2057             # This used to be our default with discard not being specified:
2058             $detectzeroes = 'on';
2059         }
2060         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2061     }
2062
2063     my $pathinfo = $path ? "file=$path," : '';
2064
2065     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2066 }
2067
2068 sub print_netdevice_full {
2069     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2070
2071     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2072
2073     my $device = $net->{model};
2074     if ($net->{model} eq 'virtio') {
2075          $device = 'virtio-net-pci';
2076      };
2077
2078     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2079     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2080     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2081         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2082         my $vectors = $net->{queues} * 2 + 2;
2083         $tmpstr .= ",vectors=$vectors,mq=on";
2084     }
2085     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2086
2087     if ($use_old_bios_files) {
2088         my $romfile;
2089         if ($device eq 'virtio-net-pci') {
2090             $romfile = 'pxe-virtio.rom';
2091         } elsif ($device eq 'e1000') {
2092             $romfile = 'pxe-e1000.rom';
2093         } elsif ($device eq 'ne2k') {
2094             $romfile = 'pxe-ne2k_pci.rom';
2095         } elsif ($device eq 'pcnet') {
2096             $romfile = 'pxe-pcnet.rom';
2097         } elsif ($device eq 'rtl8139') {
2098             $romfile = 'pxe-rtl8139.rom';
2099         }
2100         $tmpstr .= ",romfile=$romfile" if $romfile;
2101     }
2102
2103     return $tmpstr;
2104 }
2105
2106 sub print_netdev_full {
2107     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2108
2109     my $i = '';
2110     if ($netid =~ m/^net(\d+)$/) {
2111         $i = int($1);
2112     }
2113
2114     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2115
2116     my $ifname = "tap${vmid}i$i";
2117
2118     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2119     die "interface name '$ifname' is too long (max 15 character)\n"
2120         if length($ifname) >= 16;
2121
2122     my $vhostparam = '';
2123     if (is_native($arch)) {
2124         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2125     }
2126
2127     my $vmname = $conf->{name} || "vm$vmid";
2128
2129     my $netdev = "";
2130     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2131
2132     if ($net->{bridge}) {
2133         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2134     } else {
2135         $netdev = "type=user,id=$netid,hostname=$vmname";
2136     }
2137
2138     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2139
2140     return $netdev;
2141 }
2142
2143
2144 sub print_cpu_device {
2145     my ($conf, $id) = @_;
2146
2147     my $kvm = $conf->{kvm} // 1;
2148     my $cpu = $kvm ? "kvm64" : "qemu64";
2149     if (my $cputype = $conf->{cpu}) {
2150         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2151             or die "Cannot parse cpu description: $cputype\n";
2152         $cpu = $cpuconf->{cputype};
2153     }
2154
2155     my $cores = $conf->{cores} || 1;
2156
2157     my $current_core = ($id - 1) % $cores;
2158     my $current_socket = int(($id - 1 - $current_core)/$cores);
2159
2160     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2161 }
2162
2163 my $vga_map = {
2164     'cirrus' => 'cirrus-vga',
2165     'std' => 'VGA',
2166     'vmware' => 'vmware-svga',
2167     'virtio' => 'virtio-vga',
2168 };
2169
2170 sub print_vga_device {
2171     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2172
2173     my $type = $vga_map->{$vga->{type}};
2174     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2175         $type = 'virtio-gpu';
2176     }
2177     my $vgamem_mb = $vga->{memory};
2178     if ($qxlnum) {
2179         $type = $id ? 'qxl' : 'qxl-vga';
2180     }
2181     die "no devicetype for $vga->{type}\n" if !$type;
2182
2183     my $memory = "";
2184     if ($vgamem_mb) {
2185         if ($vga->{type} eq 'virtio') {
2186             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2187             $memory = ",max_hostmem=$bytes";
2188         } elsif ($qxlnum) {
2189             # from https://www.spice-space.org/multiple-monitors.html
2190             $memory = ",vgamem_mb=$vga->{memory}";
2191             my $ram = $vgamem_mb * 4;
2192             my $vram = $vgamem_mb * 2;
2193             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2194         } else {
2195             $memory = ",vgamem_mb=$vga->{memory}";
2196         }
2197     } elsif ($qxlnum && $id) {
2198         $memory = ",ram_size=67108864,vram_size=33554432";
2199     }
2200
2201     my $q35 = machine_type_is_q35($conf);
2202     my $vgaid = "vga" . ($id // '');
2203     my $pciaddr;
2204
2205     if ($q35 && $vgaid eq 'vga') {
2206         # the first display uses pcie.0 bus on q35 machines
2207         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2208     } else {
2209         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2210     }
2211
2212     return "$type,id=${vgaid}${memory}${pciaddr}";
2213 }
2214
2215 sub drive_is_cloudinit {
2216     my ($drive) = @_;
2217     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2218 }
2219
2220 sub drive_is_cdrom {
2221     my ($drive, $exclude_cloudinit) = @_;
2222
2223     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2224
2225     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2226
2227 }
2228
2229 sub parse_number_sets {
2230     my ($set) = @_;
2231     my $res = [];
2232     foreach my $part (split(/;/, $set)) {
2233         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2234             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2235             push @$res, [ $1, $2 ];
2236         } else {
2237             die "invalid range: $part\n";
2238         }
2239     }
2240     return $res;
2241 }
2242
2243 sub parse_numa {
2244     my ($data) = @_;
2245
2246     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2247     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2248     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2249     return $res;
2250 }
2251
2252 sub parse_hostpci {
2253     my ($value) = @_;
2254
2255     return undef if !$value;
2256
2257     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2258
2259     my @idlist = split(/;/, $res->{host});
2260     delete $res->{host};
2261     foreach my $id (@idlist) {
2262         if ($id =~ m/\./) { # full id 00:00.1
2263             push @{$res->{pciid}}, {
2264                 id => $id,
2265             };
2266         } else { # partial id 00:00
2267             $res->{pciid} = PVE::SysFSTools::lspci($id);
2268         }
2269     }
2270     return $res;
2271 }
2272
2273 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2274 sub parse_net {
2275     my ($data) = @_;
2276
2277     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2278     if ($@) {
2279         warn $@;
2280         return undef;
2281     }
2282     if (!defined($res->{macaddr})) {
2283         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2284         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2285     }
2286     return $res;
2287 }
2288
2289 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2290 sub parse_ipconfig {
2291     my ($data) = @_;
2292
2293     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2294     if ($@) {
2295         warn $@;
2296         return undef;
2297     }
2298
2299     if ($res->{gw} && !$res->{ip}) {
2300         warn 'gateway specified without specifying an IP address';
2301         return undef;
2302     }
2303     if ($res->{gw6} && !$res->{ip6}) {
2304         warn 'IPv6 gateway specified without specifying an IPv6 address';
2305         return undef;
2306     }
2307     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2308         warn 'gateway specified together with DHCP';
2309         return undef;
2310     }
2311     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2312         # gw6 + auto/dhcp
2313         warn "IPv6 gateway specified together with $res->{ip6} address";
2314         return undef;
2315     }
2316
2317     if (!$res->{ip} && !$res->{ip6}) {
2318         return { ip => 'dhcp', ip6 => 'dhcp' };
2319     }
2320
2321     return $res;
2322 }
2323
2324 sub print_net {
2325     my $net = shift;
2326
2327     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2328 }
2329
2330 sub add_random_macs {
2331     my ($settings) = @_;
2332
2333     foreach my $opt (keys %$settings) {
2334         next if $opt !~ m/^net(\d+)$/;
2335         my $net = parse_net($settings->{$opt});
2336         next if !$net;
2337         $settings->{$opt} = print_net($net);
2338     }
2339 }
2340
2341 sub vm_is_volid_owner {
2342     my ($storecfg, $vmid, $volid) = @_;
2343
2344     if ($volid !~  m|^/|) {
2345         my ($path, $owner);
2346         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2347         if ($owner && ($owner == $vmid)) {
2348             return 1;
2349         }
2350     }
2351
2352     return undef;
2353 }
2354
2355 sub vmconfig_register_unused_drive {
2356     my ($storecfg, $vmid, $conf, $drive) = @_;
2357
2358     if (drive_is_cloudinit($drive)) {
2359         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2360         warn $@ if $@;
2361     } elsif (!drive_is_cdrom($drive)) {
2362         my $volid = $drive->{file};
2363         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2364             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2365         }
2366     }
2367 }
2368
2369 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2370 my $smbios1_fmt = {
2371     uuid => {
2372         type => 'string',
2373         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2374         format_description => 'UUID',
2375         description => "Set SMBIOS1 UUID.",
2376         optional => 1,
2377     },
2378     version => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 version.",
2383         optional => 1,
2384     },
2385     serial => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 serial number.",
2390         optional => 1,
2391     },
2392     manufacturer => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 manufacturer.",
2397         optional => 1,
2398     },
2399     product => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 product ID.",
2404         optional => 1,
2405     },
2406     sku => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 SKU string.",
2411         optional => 1,
2412     },
2413     family => {
2414         type => 'string',
2415         pattern => '[A-Za-z0-9+\/]+={0,2}',
2416         format_description => 'Base64 encoded string',
2417         description => "Set SMBIOS1 family string.",
2418         optional => 1,
2419     },
2420     base64 => {
2421         type => 'boolean',
2422         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2423         optional => 1,
2424     },
2425 };
2426
2427 sub parse_smbios1 {
2428     my ($data) = @_;
2429
2430     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2431     warn $@ if $@;
2432     return $res;
2433 }
2434
2435 sub print_smbios1 {
2436     my ($smbios1) = @_;
2437     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2438 }
2439
2440 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2441
2442 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2443 sub verify_bootdisk {
2444     my ($value, $noerr) = @_;
2445
2446     return $value if is_valid_drivename($value);
2447
2448     return undef if $noerr;
2449
2450     die "invalid boot disk '$value'\n";
2451 }
2452
2453 sub parse_watchdog {
2454     my ($value) = @_;
2455
2456     return undef if !$value;
2457
2458     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2459     warn $@ if $@;
2460     return $res;
2461 }
2462
2463 sub parse_guest_agent {
2464     my ($value) = @_;
2465
2466     return {} if !defined($value->{agent});
2467
2468     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2469     warn $@ if $@;
2470
2471     # if the agent is disabled ignore the other potentially set properties
2472     return {} if !$res->{enabled};
2473     return $res;
2474 }
2475
2476 sub parse_vga {
2477     my ($value) = @_;
2478
2479     return {} if !$value;
2480     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2481     warn $@ if $@;
2482     return $res;
2483 }
2484
2485 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2486 sub verify_usb_device {
2487     my ($value, $noerr) = @_;
2488
2489     return $value if parse_usb_device($value);
2490
2491     return undef if $noerr;
2492
2493     die "unable to parse usb device\n";
2494 }
2495
2496 # add JSON properties for create and set function
2497 sub json_config_properties {
2498     my $prop = shift;
2499
2500     foreach my $opt (keys %$confdesc) {
2501         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2502         $prop->{$opt} = $confdesc->{$opt};
2503     }
2504
2505     return $prop;
2506 }
2507
2508 # return copy of $confdesc_cloudinit to generate documentation
2509 sub cloudinit_config_properties {
2510
2511     return dclone($confdesc_cloudinit);
2512 }
2513
2514 sub check_type {
2515     my ($key, $value) = @_;
2516
2517     die "unknown setting '$key'\n" if !$confdesc->{$key};
2518
2519     my $type = $confdesc->{$key}->{type};
2520
2521     if (!defined($value)) {
2522         die "got undefined value\n";
2523     }
2524
2525     if ($value =~ m/[\n\r]/) {
2526         die "property contains a line feed\n";
2527     }
2528
2529     if ($type eq 'boolean') {
2530         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2531         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2532         die "type check ('boolean') failed - got '$value'\n";
2533     } elsif ($type eq 'integer') {
2534         return int($1) if $value =~ m/^(\d+)$/;
2535         die "type check ('integer') failed - got '$value'\n";
2536     } elsif ($type eq 'number') {
2537         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2538         die "type check ('number') failed - got '$value'\n";
2539     } elsif ($type eq 'string') {
2540         if (my $fmt = $confdesc->{$key}->{format}) {
2541             PVE::JSONSchema::check_format($fmt, $value);
2542             return $value;
2543         }
2544         $value =~ s/^\"(.*)\"$/$1/;
2545         return $value;
2546     } else {
2547         die "internal error"
2548     }
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conf = PVE::QemuConfig->load_config($vmid);
2555
2556     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2557
2558     if ($conf->{template}) {
2559         # check if any base image is still used by a linked clone
2560         foreach_drive($conf, sub {
2561                 my ($ds, $drive) = @_;
2562
2563                 return if drive_is_cdrom($drive);
2564
2565                 my $volid = $drive->{file};
2566
2567                 return if !$volid || $volid =~ m|^/|;
2568
2569                 die "base volume '$volid' is still in use by linked cloned\n"
2570                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2571
2572         });
2573     }
2574
2575     # only remove disks owned by this VM
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578
2579         return if drive_is_cdrom($drive, 1);
2580
2581         my $volid = $drive->{file};
2582
2583         return if !$volid || $volid =~ m|^/|;
2584
2585         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2586         return if !$path || !$owner || ($owner != $vmid);
2587
2588         eval {
2589             PVE::Storage::vdisk_free($storecfg, $volid);
2590         };
2591         warn "Could not remove disk '$volid', check manually: $@" if $@;
2592
2593     });
2594
2595     # also remove unused disk
2596     eval {
2597         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2598
2599         eval {
2600             PVE::Storage::foreach_volid($dl, sub {
2601                 my ($volid, $sid, $volname, $d) = @_;
2602                 PVE::Storage::vdisk_free($storecfg, $volid);
2603             });
2604         };
2605         warn $@ if $@;
2606
2607     };
2608     warn $@ if $@;
2609
2610     if ($keep_empty_config) {
2611         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2612     } else {
2613         PVE::QemuConfig->destroy_config($vmid);
2614     }
2615 }
2616
2617 sub parse_vm_config {
2618     my ($filename, $raw) = @_;
2619
2620     return undef if !defined($raw);
2621
2622     my $res = {
2623         digest => Digest::SHA::sha1_hex($raw),
2624         snapshots => {},
2625         pending => {},
2626     };
2627
2628     $filename =~ m|/qemu-server/(\d+)\.conf$|
2629         || die "got strange filename '$filename'";
2630
2631     my $vmid = $1;
2632
2633     my $conf = $res;
2634     my $descr;
2635     my $section = '';
2636
2637     my @lines = split(/\n/, $raw);
2638     foreach my $line (@lines) {
2639         next if $line =~ m/^\s*$/;
2640
2641         if ($line =~ m/^\[PENDING\]\s*$/i) {
2642             $section = 'pending';
2643             if (defined($descr)) {
2644                 $descr =~ s/\s+$//;
2645                 $conf->{description} = $descr;
2646             }
2647             $descr = undef;
2648             $conf = $res->{$section} = {};
2649             next;
2650
2651         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2652             $section = $1;
2653             if (defined($descr)) {
2654                 $descr =~ s/\s+$//;
2655                 $conf->{description} = $descr;
2656             }
2657             $descr = undef;
2658             $conf = $res->{snapshots}->{$section} = {};
2659             next;
2660         }
2661
2662         if ($line =~ m/^\#(.*)\s*$/) {
2663             $descr = '' if !defined($descr);
2664             $descr .= PVE::Tools::decode_text($1) . "\n";
2665             next;
2666         }
2667
2668         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2669             $descr = '' if !defined($descr);
2670             $descr .= PVE::Tools::decode_text($2);
2671         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2672             $conf->{snapstate} = $1;
2673         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2674             my $key = $1;
2675             my $value = $2;
2676             $conf->{$key} = $value;
2677         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2678             my $value = $1;
2679             if ($section eq 'pending') {
2680                 $conf->{delete} = $value; # we parse this later
2681             } else {
2682                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2683             }
2684         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2685             my $key = $1;
2686             my $value = $2;
2687             eval { $value = check_type($key, $value); };
2688             if ($@) {
2689                 warn "vm $vmid - unable to parse value of '$key' - $@";
2690             } else {
2691                 $key = 'ide2' if $key eq 'cdrom';
2692                 my $fmt = $confdesc->{$key}->{format};
2693                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2694                     my $v = parse_drive($key, $value);
2695                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2696                         $v->{file} = $volid;
2697                         $value = print_drive($vmid, $v);
2698                     } else {
2699                         warn "vm $vmid - unable to parse value of '$key'\n";
2700                         next;
2701                     }
2702                 }
2703
2704                 $conf->{$key} = $value;
2705             }
2706         }
2707     }
2708
2709     if (defined($descr)) {
2710         $descr =~ s/\s+$//;
2711         $conf->{description} = $descr;
2712     }
2713     delete $res->{snapstate}; # just to be sure
2714
2715     return $res;
2716 }
2717
2718 sub write_vm_config {
2719     my ($filename, $conf) = @_;
2720
2721     delete $conf->{snapstate}; # just to be sure
2722
2723     if ($conf->{cdrom}) {
2724         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2725         $conf->{ide2} = $conf->{cdrom};
2726         delete $conf->{cdrom};
2727     }
2728
2729     # we do not use 'smp' any longer
2730     if ($conf->{sockets}) {
2731         delete $conf->{smp};
2732     } elsif ($conf->{smp}) {
2733         $conf->{sockets} = $conf->{smp};
2734         delete $conf->{cores};
2735         delete $conf->{smp};
2736     }
2737
2738     my $used_volids = {};
2739
2740     my $cleanup_config = sub {
2741         my ($cref, $pending, $snapname) = @_;
2742
2743         foreach my $key (keys %$cref) {
2744             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2745                 $key eq 'snapstate' || $key eq 'pending';
2746             my $value = $cref->{$key};
2747             if ($key eq 'delete') {
2748                 die "propertry 'delete' is only allowed in [PENDING]\n"
2749                     if !$pending;
2750                 # fixme: check syntax?
2751                 next;
2752             }
2753             eval { $value = check_type($key, $value); };
2754             die "unable to parse value of '$key' - $@" if $@;
2755
2756             $cref->{$key} = $value;
2757
2758             if (!$snapname && is_valid_drivename($key)) {
2759                 my $drive = parse_drive($key, $value);
2760                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2761             }
2762         }
2763     };
2764
2765     &$cleanup_config($conf);
2766
2767     &$cleanup_config($conf->{pending}, 1);
2768
2769     foreach my $snapname (keys %{$conf->{snapshots}}) {
2770         die "internal error" if $snapname eq 'pending';
2771         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2772     }
2773
2774     # remove 'unusedX' settings if we re-add a volume
2775     foreach my $key (keys %$conf) {
2776         my $value = $conf->{$key};
2777         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2778             delete $conf->{$key};
2779         }
2780     }
2781
2782     my $generate_raw_config = sub {
2783         my ($conf, $pending) = @_;
2784
2785         my $raw = '';
2786
2787         # add description as comment to top of file
2788         if (defined(my $descr = $conf->{description})) {
2789             if ($descr) {
2790                 foreach my $cl (split(/\n/, $descr)) {
2791                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2792                 }
2793             } else {
2794                 $raw .= "#\n" if $pending;
2795             }
2796         }
2797
2798         foreach my $key (sort keys %$conf) {
2799             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2800             $raw .= "$key: $conf->{$key}\n";
2801         }
2802         return $raw;
2803     };
2804
2805     my $raw = &$generate_raw_config($conf);
2806
2807     if (scalar(keys %{$conf->{pending}})){
2808         $raw .= "\n[PENDING]\n";
2809         $raw .= &$generate_raw_config($conf->{pending}, 1);
2810     }
2811
2812     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2813         $raw .= "\n[$snapname]\n";
2814         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2815     }
2816
2817     return $raw;
2818 }
2819
2820 sub load_defaults {
2821
2822     my $res = {};
2823
2824     # we use static defaults from our JSON schema configuration
2825     foreach my $key (keys %$confdesc) {
2826         if (defined(my $default = $confdesc->{$key}->{default})) {
2827             $res->{$key} = $default;
2828         }
2829     }
2830
2831     return $res;
2832 }
2833
2834 sub config_list {
2835     my $vmlist = PVE::Cluster::get_vmlist();
2836     my $res = {};
2837     return $res if !$vmlist || !$vmlist->{ids};
2838     my $ids = $vmlist->{ids};
2839
2840     foreach my $vmid (keys %$ids) {
2841         my $d = $ids->{$vmid};
2842         next if !$d->{node} || $d->{node} ne $nodename;
2843         next if !$d->{type} || $d->{type} ne 'qemu';
2844         $res->{$vmid}->{exists} = 1;
2845     }
2846     return $res;
2847 }
2848
2849 # test if VM uses local resources (to prevent migration)
2850 sub check_local_resources {
2851     my ($conf, $noerr) = @_;
2852
2853     my @loc_res = ();
2854
2855     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2856     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2857
2858     push @loc_res, "ivshmem" if $conf->{ivshmem};
2859
2860     foreach my $k (keys %$conf) {
2861         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2862         # sockets are safe: they will recreated be on the target side post-migrate
2863         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2864         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2865     }
2866
2867     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2868
2869     return \@loc_res;
2870 }
2871
2872 # check if used storages are available on all nodes (use by migrate)
2873 sub check_storage_availability {
2874     my ($storecfg, $conf, $node) = @_;
2875
2876     foreach_drive($conf, sub {
2877         my ($ds, $drive) = @_;
2878
2879         my $volid = $drive->{file};
2880         return if !$volid;
2881
2882         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2883         return if !$sid;
2884
2885         # check if storage is available on both nodes
2886         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2887         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2888    });
2889 }
2890
2891 # list nodes where all VM images are available (used by has_feature API)
2892 sub shared_nodes {
2893     my ($conf, $storecfg) = @_;
2894
2895     my $nodelist = PVE::Cluster::get_nodelist();
2896     my $nodehash = { map { $_ => 1 } @$nodelist };
2897     my $nodename = PVE::INotify::nodename();
2898
2899     foreach_drive($conf, sub {
2900         my ($ds, $drive) = @_;
2901
2902         my $volid = $drive->{file};
2903         return if !$volid;
2904
2905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906         if ($storeid) {
2907             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2908             if ($scfg->{disable}) {
2909                 $nodehash = {};
2910             } elsif (my $avail = $scfg->{nodes}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     delete $nodehash->{$node} if !$avail->{$node};
2913                 }
2914             } elsif (!$scfg->{shared}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     delete $nodehash->{$node} if $node ne $nodename
2917                 }
2918             }
2919         }
2920     });
2921
2922     return $nodehash
2923 }
2924
2925 sub check_local_storage_availability {
2926     my ($conf, $storecfg) = @_;
2927
2928     my $nodelist = PVE::Cluster::get_nodelist();
2929     my $nodehash = { map { $_ => {} } @$nodelist };
2930
2931     foreach_drive($conf, sub {
2932         my ($ds, $drive) = @_;
2933
2934         my $volid = $drive->{file};
2935         return if !$volid;
2936
2937         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2938         if ($storeid) {
2939             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2940
2941             if ($scfg->{disable}) {
2942                 foreach my $node (keys %$nodehash) {
2943                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2944                 }
2945             } elsif (my $avail = $scfg->{nodes}) {
2946                 foreach my $node (keys %$nodehash) {
2947                     if (!$avail->{$node}) {
2948                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2949                     }
2950                 }
2951             }
2952         }
2953     });
2954
2955     foreach my $node (values %$nodehash) {
2956         if (my $unavail = $node->{unavailable_storages}) {
2957             $node->{unavailable_storages} = [ sort keys %$unavail ];
2958         }
2959     }
2960
2961     return $nodehash
2962 }
2963
2964 sub check_cmdline {
2965     my ($pidfile, $pid) = @_;
2966
2967     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2968     if (defined($fh)) {
2969         my $line = <$fh>;
2970         $fh->close;
2971         return undef if !$line;
2972         my @param = split(/\0/, $line);
2973
2974         my $cmd = $param[0];
2975         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2976
2977         for (my $i = 0; $i < scalar (@param); $i++) {
2978             my $p = $param[$i];
2979             next if !$p;
2980             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2981                 my $p = $param[$i+1];
2982                 return 1 if $p && ($p eq $pidfile);
2983                 return undef;
2984             }
2985         }
2986     }
2987     return undef;
2988 }
2989
2990 sub check_running {
2991     my ($vmid, $nocheck, $node) = @_;
2992
2993     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2994
2995     die "unable to find configuration file for VM $vmid - no such machine\n"
2996         if !$nocheck && ! -f $filename;
2997
2998     my $pidfile = pidfile_name($vmid);
2999
3000     if (my $fd = IO::File->new("<$pidfile")) {
3001         my $st = stat($fd);
3002         my $line = <$fd>;
3003         close($fd);
3004
3005         my $mtime = $st->mtime;
3006         if ($mtime > time()) {
3007             warn "file '$filename' modified in future\n";
3008         }
3009
3010         if ($line =~ m/^(\d+)$/) {
3011             my $pid = $1;
3012             if (check_cmdline($pidfile, $pid)) {
3013                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3014                     return $pid;
3015                 }
3016             }
3017         }
3018     }
3019
3020     return undef;
3021 }
3022
3023 sub vzlist {
3024
3025     my $vzlist = config_list();
3026
3027     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3028
3029     while (defined(my $de = $fd->read)) {
3030         next if $de !~ m/^(\d+)\.pid$/;
3031         my $vmid = $1;
3032         next if !defined($vzlist->{$vmid});
3033         if (my $pid = check_running($vmid)) {
3034             $vzlist->{$vmid}->{pid} = $pid;
3035         }
3036     }
3037
3038     return $vzlist;
3039 }
3040
3041 sub disksize {
3042     my ($storecfg, $conf) = @_;
3043
3044     my $bootdisk = $conf->{bootdisk};
3045     return undef if !$bootdisk;
3046     return undef if !is_valid_drivename($bootdisk);
3047
3048     return undef if !$conf->{$bootdisk};
3049
3050     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3051     return undef if !defined($drive);
3052
3053     return undef if drive_is_cdrom($drive);
3054
3055     my $volid = $drive->{file};
3056     return undef if !$volid;
3057
3058     return $drive->{size};
3059 }
3060
3061 our $vmstatus_return_properties = {
3062     vmid => get_standard_option('pve-vmid'),
3063     status => {
3064         description => "Qemu process status.",
3065         type => 'string',
3066         enum => ['stopped', 'running'],
3067     },
3068     maxmem => {
3069         description => "Maximum memory in bytes.",
3070         type => 'integer',
3071         optional => 1,
3072         renderer => 'bytes',
3073     },
3074     maxdisk => {
3075         description => "Root disk size in bytes.",
3076         type => 'integer',
3077         optional => 1,
3078         renderer => 'bytes',
3079     },
3080     name => {
3081         description => "VM name.",
3082         type => 'string',
3083         optional => 1,
3084     },
3085     qmpstatus => {
3086         description => "Qemu QMP agent status.",
3087         type => 'string',
3088         optional => 1,
3089     },
3090     pid => {
3091         description => "PID of running qemu process.",
3092         type => 'integer',
3093         optional => 1,
3094     },
3095     uptime => {
3096         description => "Uptime.",
3097         type => 'integer',
3098         optional => 1,
3099         renderer => 'duration',
3100     },
3101     cpus => {
3102         description => "Maximum usable CPUs.",
3103         type => 'number',
3104         optional => 1,
3105     },
3106     lock => {
3107         description => "The current config lock, if any.",
3108         type => 'string',
3109         optional => 1,
3110     }
3111 };
3112
3113 my $last_proc_pid_stat;
3114
3115 # get VM status information
3116 # This must be fast and should not block ($full == false)
3117 # We only query KVM using QMP if $full == true (this can be slow)
3118 sub vmstatus {
3119     my ($opt_vmid, $full) = @_;
3120
3121     my $res = {};
3122
3123     my $storecfg = PVE::Storage::config();
3124
3125     my $list = vzlist();
3126     my $defaults = load_defaults();
3127
3128     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3129
3130     my $cpucount = $cpuinfo->{cpus} || 1;
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134
3135         my $conf = PVE::QemuConfig->load_config($vmid);
3136
3137         my $d = { vmid => $vmid };
3138         $d->{pid} = $list->{$vmid}->{pid};
3139
3140         # fixme: better status?
3141         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3142
3143         my $size = disksize($storecfg, $conf);
3144         if (defined($size)) {
3145             $d->{disk} = 0; # no info available
3146             $d->{maxdisk} = $size;
3147         } else {
3148             $d->{disk} = 0;
3149             $d->{maxdisk} = 0;
3150         }
3151
3152         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3153             * ($conf->{cores} || $defaults->{cores});
3154         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3155         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3156
3157         $d->{name} = $conf->{name} || "VM $vmid";
3158         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3159             : $defaults->{memory}*(1024*1024);
3160
3161         if ($conf->{balloon}) {
3162             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3163             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3164                 : $defaults->{shares};
3165         }
3166
3167         $d->{uptime} = 0;
3168         $d->{cpu} = 0;
3169         $d->{mem} = 0;
3170
3171         $d->{netout} = 0;
3172         $d->{netin} = 0;
3173
3174         $d->{diskread} = 0;
3175         $d->{diskwrite} = 0;
3176
3177         $d->{template} = PVE::QemuConfig->is_template($conf);
3178
3179         $d->{serial} = 1 if conf_has_serial($conf);
3180         $d->{lock} = $conf->{lock} if $conf->{lock};
3181
3182         $res->{$vmid} = $d;
3183     }
3184
3185     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3186     foreach my $dev (keys %$netdev) {
3187         next if $dev !~ m/^tap([1-9]\d*)i/;
3188         my $vmid = $1;
3189         my $d = $res->{$vmid};
3190         next if !$d;
3191
3192         $d->{netout} += $netdev->{$dev}->{receive};
3193         $d->{netin} += $netdev->{$dev}->{transmit};
3194
3195         if ($full) {
3196             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3197             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3198         }
3199
3200     }
3201
3202     my $ctime = gettimeofday;
3203
3204     foreach my $vmid (keys %$list) {
3205
3206         my $d = $res->{$vmid};
3207         my $pid = $d->{pid};
3208         next if !$pid;
3209
3210         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3211         next if !$pstat; # not running
3212
3213         my $used = $pstat->{utime} + $pstat->{stime};
3214
3215         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3216
3217         if ($pstat->{vsize}) {
3218             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3219         }
3220
3221         my $old = $last_proc_pid_stat->{$pid};
3222         if (!$old) {
3223             $last_proc_pid_stat->{$pid} = {
3224                 time => $ctime,
3225                 used => $used,
3226                 cpu => 0,
3227             };
3228             next;
3229         }
3230
3231         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3232
3233         if ($dtime > 1000) {
3234             my $dutime = $used -  $old->{used};
3235
3236             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3237             $last_proc_pid_stat->{$pid} = {
3238                 time => $ctime,
3239                 used => $used,
3240                 cpu => $d->{cpu},
3241             };
3242         } else {
3243             $d->{cpu} = $old->{cpu};
3244         }
3245     }
3246
3247     return $res if !$full;
3248
3249     my $qmpclient = PVE::QMPClient->new();
3250
3251     my $ballooncb = sub {
3252         my ($vmid, $resp) = @_;
3253
3254         my $info = $resp->{'return'};
3255         return if !$info->{max_mem};
3256
3257         my $d = $res->{$vmid};
3258
3259         # use memory assigned to VM
3260         $d->{maxmem} = $info->{max_mem};
3261         $d->{balloon} = $info->{actual};
3262
3263         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3264             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3265             $d->{freemem} = $info->{free_mem};
3266         }
3267
3268         $d->{ballooninfo} = $info;
3269     };
3270
3271     my $blockstatscb = sub {
3272         my ($vmid, $resp) = @_;
3273         my $data = $resp->{'return'} || [];
3274         my $totalrdbytes = 0;
3275         my $totalwrbytes = 0;
3276
3277         for my $blockstat (@$data) {
3278             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3279             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3280
3281             $blockstat->{device} =~ s/drive-//;
3282             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3283         }
3284         $res->{$vmid}->{diskread} = $totalrdbytes;
3285         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3286     };
3287
3288     my $statuscb = sub {
3289         my ($vmid, $resp) = @_;
3290
3291         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3292         # this fails if ballon driver is not loaded, so this must be
3293         # the last commnand (following command are aborted if this fails).
3294         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3295
3296         my $status = 'unknown';
3297         if (!defined($status = $resp->{'return'}->{status})) {
3298             warn "unable to get VM status\n";
3299             return;
3300         }
3301
3302         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3303     };
3304
3305     foreach my $vmid (keys %$list) {
3306         next if $opt_vmid && ($vmid ne $opt_vmid);
3307         next if !$res->{$vmid}->{pid}; # not running
3308         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3309     }
3310
3311     $qmpclient->queue_execute(undef, 2);
3312
3313     foreach my $vmid (keys %$list) {
3314         next if $opt_vmid && ($vmid ne $opt_vmid);
3315         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3316     }
3317
3318     return $res;
3319 }
3320
3321 sub foreach_drive {
3322     my ($conf, $func, @param) = @_;
3323
3324     foreach my $ds (valid_drive_names()) {
3325         next if !defined($conf->{$ds});
3326
3327         my $drive = parse_drive($ds, $conf->{$ds});
3328         next if !$drive;
3329
3330         &$func($ds, $drive, @param);
3331     }
3332 }
3333
3334 sub foreach_volid {
3335     my ($conf, $func, @param) = @_;
3336
3337     my $volhash = {};
3338
3339     my $test_volid = sub {
3340         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3341
3342         return if !$volid;
3343
3344         $volhash->{$volid}->{cdrom} //= 1;
3345         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3346
3347         $volhash->{$volid}->{replicate} //= 0;
3348         $volhash->{$volid}->{replicate} = 1 if $replicate;
3349
3350         $volhash->{$volid}->{shared} //= 0;
3351         $volhash->{$volid}->{shared} = 1 if $shared;
3352
3353         $volhash->{$volid}->{referenced_in_config} //= 0;
3354         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3355
3356         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3357             if defined($snapname);
3358         $volhash->{$volid}->{size} = $size if $size;
3359     };
3360
3361     foreach_drive($conf, sub {
3362         my ($ds, $drive) = @_;
3363         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3364     });
3365
3366     foreach my $snapname (keys %{$conf->{snapshots}}) {
3367         my $snap = $conf->{snapshots}->{$snapname};
3368         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3369         foreach_drive($snap, sub {
3370             my ($ds, $drive) = @_;
3371             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3372         });
3373     }
3374
3375     foreach my $volid (keys %$volhash) {
3376         &$func($volid, $volhash->{$volid}, @param);
3377     }
3378 }
3379
3380 sub conf_has_serial {
3381     my ($conf) = @_;
3382
3383     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3384         if ($conf->{"serial$i"}) {
3385             return 1;
3386         }
3387     }
3388
3389     return 0;
3390 }
3391
3392 sub conf_has_audio {
3393     my ($conf, $id) = @_;
3394
3395     $id //= 0;
3396     my $audio = $conf->{"audio$id"};
3397     return undef if !defined($audio);
3398
3399     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3400     my $audiodriver = $audioproperties->{driver} // 'spice';
3401
3402     return {
3403         dev => $audioproperties->{device},
3404         dev_id => "audiodev$id",
3405         backend => $audiodriver,
3406         backend_id => "$audiodriver-backend${id}",
3407     };
3408 }
3409
3410 sub vga_conf_has_spice {
3411     my ($vga) = @_;
3412
3413     my $vgaconf = parse_vga($vga);
3414     my $vgatype = $vgaconf->{type};
3415     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3416
3417     return $1 || 1;
3418 }
3419
3420 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3421 sub get_host_arch() {
3422     $host_arch = (POSIX::uname())[4] if !$host_arch;
3423     return $host_arch;
3424 }
3425
3426 sub is_native($) {
3427     my ($arch) = @_;
3428     return get_host_arch() eq $arch;
3429 }
3430
3431 my $default_machines = {
3432     x86_64 => 'pc',
3433     aarch64 => 'virt',
3434 };
3435
3436 sub get_basic_machine_info {
3437     my ($conf, $forcemachine) = @_;
3438
3439     my $arch = $conf->{arch} // get_host_arch();
3440     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3441     return ($arch, $machine);
3442 }
3443
3444 sub get_ovmf_files($) {
3445     my ($arch) = @_;
3446
3447     my $ovmf = $OVMF->{$arch}
3448         or die "no OVMF images known for architecture '$arch'\n";
3449
3450     return @$ovmf;
3451 }
3452
3453 my $Arch2Qemu = {
3454     aarch64 => '/usr/bin/qemu-system-aarch64',
3455     x86_64 => '/usr/bin/qemu-system-x86_64',
3456 };
3457 sub get_command_for_arch($) {
3458     my ($arch) = @_;
3459     return '/usr/bin/kvm' if is_native($arch);
3460
3461     my $cmd = $Arch2Qemu->{$arch}
3462         or die "don't know how to emulate architecture '$arch'\n";
3463     return $cmd;
3464 }
3465
3466 sub get_cpu_options {
3467     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3468
3469     my $cpuFlags = [];
3470     my $ostype = $conf->{ostype};
3471
3472     my $cpu = $kvm ? "kvm64" : "qemu64";
3473     if ($arch eq 'aarch64') {
3474         $cpu = 'cortex-a57';
3475     }
3476     my $hv_vendor_id;
3477     if (my $cputype = $conf->{cpu}) {
3478         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3479             or die "Cannot parse cpu description: $cputype\n";
3480         $cpu = $cpuconf->{cputype};
3481         $kvm_off = 1 if $cpuconf->{hidden};
3482         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3483
3484         if (defined(my $flags = $cpuconf->{flags})) {
3485             push @$cpuFlags, split(";", $flags);
3486         }
3487     }
3488
3489     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3490
3491     push @$cpuFlags , '-x2apic'
3492         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3493
3494     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3495
3496     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3497
3498     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3499
3500         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3501         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3502     }
3503
3504     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3505
3506     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3507
3508     push @$cpuFlags, 'kvm=off' if $kvm_off;
3509
3510     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3511         push @$cpuFlags, "vendor=${cpu_vendor}"
3512             if $cpu_vendor ne 'default';
3513     } elsif ($arch ne 'aarch64') {
3514         die "internal error"; # should not happen
3515     }
3516
3517     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3518
3519     return ('-cpu', $cpu);
3520 }
3521
3522 sub config_to_command {
3523     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3524
3525     my $cmd = [];
3526     my $globalFlags = [];
3527     my $machineFlags = [];
3528     my $rtcFlags = [];
3529     my $devices = [];
3530     my $pciaddr = '';
3531     my $bridges = {};
3532     my $vernum = 0; # unknown
3533     my $ostype = $conf->{ostype};
3534     my $winversion = windows_version($ostype);
3535     my $kvm = $conf->{kvm};
3536
3537     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3538     my $kvm_binary = get_command_for_arch($arch);
3539     my $kvmver = kvm_user_version($kvm_binary);
3540     $kvm //= 1 if is_native($arch);
3541
3542     if ($kvm) {
3543         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3544             if !defined kvm_version();
3545     }
3546
3547     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3548         $vernum = $1*1000000+$2*1000;
3549     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3550         $vernum = $1*1000000+$2*1000+$3;
3551     }
3552
3553     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3554
3555     my $q35 = machine_type_is_q35($conf);
3556     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3557     my $use_old_bios_files = undef;
3558     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3559
3560     my $cpuunits = defined($conf->{cpuunits}) ?
3561             $conf->{cpuunits} : $defaults->{cpuunits};
3562
3563     push @$cmd, $kvm_binary;
3564
3565     push @$cmd, '-id', $vmid;
3566
3567     my $vmname = $conf->{name} || "vm$vmid";
3568
3569     push @$cmd, '-name', $vmname;
3570
3571     my $use_virtio = 0;
3572
3573     my $qmpsocket = qmp_socket($vmid);
3574     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3575     push @$cmd, '-mon', "chardev=qmp,mode=control";
3576
3577     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3578         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3579         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3580     }
3581
3582     push @$cmd, '-pidfile' , pidfile_name($vmid);
3583
3584     push @$cmd, '-daemonize';
3585
3586     if ($conf->{smbios1}) {
3587         my $smbios_conf = parse_smbios1($conf->{smbios1});
3588         if ($smbios_conf->{base64}) {
3589             # Do not pass base64 flag to qemu
3590             delete $smbios_conf->{base64};
3591             my $smbios_string = "";
3592             foreach my $key (keys %$smbios_conf) {
3593                 my $value;
3594                 if ($key eq "uuid") {
3595                     $value = $smbios_conf->{uuid}
3596                 } else {
3597                     $value = decode_base64($smbios_conf->{$key});
3598                 }
3599                 # qemu accepts any binary data, only commas need escaping by double comma
3600                 $value =~ s/,/,,/g;
3601                 $smbios_string .= "," . $key . "=" . $value if $value;
3602             }
3603             push @$cmd, '-smbios', "type=1" . $smbios_string;
3604         } else {
3605             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3606         }
3607     }
3608
3609     if ($conf->{vmgenid}) {
3610         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3611     }
3612
3613     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3614     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3615         die "uefi base image not found\n" if ! -f $ovmf_code;
3616
3617         my $path;
3618         my $format;
3619         if (my $efidisk = $conf->{efidisk0}) {
3620             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3621             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3622             $format = $d->{format};
3623             if ($storeid) {
3624                 $path = PVE::Storage::path($storecfg, $d->{file});
3625                 if (!defined($format)) {
3626                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3627                     $format = qemu_img_format($scfg, $volname);
3628                 }
3629             } else {
3630                 $path = $d->{file};
3631                 die "efidisk format must be specified\n"
3632                     if !defined($format);
3633             }
3634         } else {
3635             warn "no efidisk configured! Using temporary efivars disk.\n";
3636             $path = "/tmp/$vmid-ovmf.fd";
3637             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3638             $format = 'raw';
3639         }
3640
3641         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3642         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3643     }
3644
3645     # load q35 config
3646     if ($q35) {
3647         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3648         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3649             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3650         } else {
3651             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3652         }
3653     }
3654
3655     # add usb controllers
3656     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3657     push @$devices, @usbcontrollers if @usbcontrollers;
3658     my $vga = parse_vga($conf->{vga});
3659
3660     my $qxlnum = vga_conf_has_spice($conf->{vga});
3661     $vga->{type} = 'qxl' if $qxlnum;
3662
3663     if (!$vga->{type}) {
3664         if ($arch eq 'aarch64') {
3665             $vga->{type} = 'virtio';
3666         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3667             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3668         } else {
3669             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3670         }
3671     }
3672
3673     # enable absolute mouse coordinates (needed by vnc)
3674     my $tablet;
3675     if (defined($conf->{tablet})) {
3676         $tablet = $conf->{tablet};
3677     } else {
3678         $tablet = $defaults->{tablet};
3679         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3680         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3681     }
3682
3683     if ($tablet) {
3684         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3685         my $kbd = print_keyboarddevice_full($conf, $arch);
3686         push @$devices, '-device', $kbd if defined($kbd);
3687     }
3688
3689     my $kvm_off = 0;
3690     my $gpu_passthrough;
3691
3692     # host pci devices
3693     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3694         my $id = "hostpci$i";
3695         my $d = parse_hostpci($conf->{$id});
3696         next if !$d;
3697
3698         if (my $pcie = $d->{pcie}) {
3699             die "q35 machine model is not enabled" if !$q35;
3700             # win7 wants to have the pcie devices directly on the pcie bus
3701             # instead of in the root port
3702             if ($winversion == 7) {
3703                 $pciaddr = print_pcie_addr("${id}bus0");
3704             } else {
3705                 # add more root ports if needed, 4 are present by default
3706                 # by pve-q35 cfgs, rest added here on demand.
3707                 if ($i > 3) {
3708                     push @$devices, '-device', print_pcie_root_port($i);
3709                 }
3710                 $pciaddr = print_pcie_addr($id);
3711             }
3712         } else {
3713             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3714         }
3715
3716         my $xvga = '';
3717         if ($d->{'x-vga'}) {
3718             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3719             $kvm_off = 1;
3720             $vga->{type} = 'none' if !defined($conf->{vga});
3721             $gpu_passthrough = 1;
3722         }
3723
3724         my $pcidevices = $d->{pciid};
3725         my $multifunction = 1 if @$pcidevices > 1;
3726
3727         my $sysfspath;
3728         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3729             my $pci_id = $pcidevices->[0]->{id};
3730             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3731             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3732         } elsif ($d->{mdev}) {
3733             warn "ignoring mediated device '$id' with multifunction device\n";
3734         }
3735
3736         my $j=0;
3737         foreach my $pcidevice (@$pcidevices) {
3738             my $devicestr = "vfio-pci";
3739
3740             if ($sysfspath) {
3741                 $devicestr .= ",sysfsdev=$sysfspath";
3742             } else {
3743                 $devicestr .= ",host=$pcidevice->{id}";
3744             }
3745
3746             my $mf_addr = $multifunction ? ".$j" : '';
3747             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3748
3749             if ($j == 0) {
3750                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3751                 $devicestr .= "$xvga";
3752                 $devicestr .= ",multifunction=on" if $multifunction;
3753                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3754             }
3755
3756             push @$devices, '-device', $devicestr;
3757             $j++;
3758         }
3759     }
3760
3761     # usb devices
3762     my $usb_dev_features = {};
3763     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3764
3765     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3766     push @$devices, @usbdevices if @usbdevices;
3767     # serial devices
3768     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3769         if (my $path = $conf->{"serial$i"}) {
3770             if ($path eq 'socket') {
3771                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3772                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3773                 # On aarch64, serial0 is the UART device. Qemu only allows
3774                 # connecting UART devices via the '-serial' command line, as
3775                 # the device has a fixed slot on the hardware...
3776                 if ($arch eq 'aarch64' && $i == 0) {
3777                     push @$devices, '-serial', "chardev:serial$i";
3778                 } else {
3779                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3780                 }
3781             } else {
3782                 die "no such serial device\n" if ! -c $path;
3783                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3784                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3785             }
3786         }
3787     }
3788
3789     # parallel devices
3790     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3791         if (my $path = $conf->{"parallel$i"}) {
3792             die "no such parallel device\n" if ! -c $path;
3793             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3794             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3795             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3796         }
3797     }
3798
3799     if (my $audio = conf_has_audio($conf)) {
3800
3801         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3802
3803         my $id = $audio->{dev_id};
3804         if ($audio->{dev} eq 'AC97') {
3805             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3806         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3807             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3808             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3809             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3810         } else {
3811             die "unkown audio device '$audio->{dev}', implement me!";
3812         }
3813
3814         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3815     }
3816
3817     my $sockets = 1;
3818     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3819     $sockets = $conf->{sockets} if  $conf->{sockets};
3820
3821     my $cores = $conf->{cores} || 1;
3822
3823     my $maxcpus = $sockets * $cores;
3824
3825     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3826
3827     my $allowed_vcpus = $cpuinfo->{cpus};
3828
3829     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3830         if ($allowed_vcpus < $maxcpus);
3831
3832     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3833
3834         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3835         for (my $i = 2; $i <= $vcpus; $i++)  {
3836             my $cpustr = print_cpu_device($conf,$i);
3837             push @$cmd, '-device', $cpustr;
3838         }
3839
3840     } else {
3841
3842         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3843     }
3844     push @$cmd, '-nodefaults';
3845
3846     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3847
3848     my $bootindex_hash = {};
3849     my $i = 1;
3850     foreach my $o (split(//, $bootorder)) {
3851         $bootindex_hash->{$o} = $i*100;
3852         $i++;
3853     }
3854
3855     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3856
3857     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3858
3859     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3860
3861     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3862         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3863         my $socket = vnc_socket($vmid);
3864         push @$cmd,  '-vnc', "unix:$socket,password";
3865     } else {
3866         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3867         push @$cmd, '-nographic';
3868     }
3869
3870     # time drift fix
3871     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3872
3873     my $useLocaltime = $conf->{localtime};
3874
3875     if ($winversion >= 5) { # windows
3876         $useLocaltime = 1 if !defined($conf->{localtime});
3877
3878         # use time drift fix when acpi is enabled
3879         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3880             $tdf = 1 if !defined($conf->{tdf});
3881         }
3882     }
3883
3884     if ($winversion >= 6) {
3885         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3886         push @$cmd, '-no-hpet';
3887     }
3888
3889     push @$rtcFlags, 'driftfix=slew' if $tdf;
3890
3891     if (!$kvm) {
3892         push @$machineFlags, 'accel=tcg';
3893     }
3894
3895     if ($machine_type) {
3896         push @$machineFlags, "type=${machine_type}";
3897     }
3898
3899     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3900         push @$rtcFlags, "base=$conf->{startdate}";
3901     } elsif ($useLocaltime) {
3902         push @$rtcFlags, 'base=localtime';
3903     }
3904
3905     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3906
3907     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3908
3909     push @$cmd, '-S' if $conf->{freeze};
3910
3911     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3912
3913     if (parse_guest_agent($conf)->{enabled}) {
3914         my $qgasocket = qmp_socket($vmid, 1);
3915         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3916         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3917         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3918         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3919     }
3920
3921     my $spice_port;
3922
3923     if ($qxlnum) {
3924         if ($qxlnum > 1) {
3925             if ($winversion){
3926                 for(my $i = 1; $i < $qxlnum; $i++){
3927                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3928                 }
3929             } else {
3930                 # assume other OS works like Linux
3931                 my ($ram, $vram) = ("134217728", "67108864");
3932                 if ($vga->{memory}) {
3933                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3934                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3935                 }
3936                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3937                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3938             }
3939         }
3940
3941         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3942
3943         my $nodename = PVE::INotify::nodename();
3944         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3945         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3946         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3947
3948         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3949         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3950         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3951
3952         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3953         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3954
3955         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3956         if ($spice_enhancement->{foldersharing}) {
3957             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3958             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3959         }
3960
3961         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3962         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3963         push @$devices, '-spice', "$spice_opts";
3964     }
3965
3966     # enable balloon by default, unless explicitly disabled
3967     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3968         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3969         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3970     }
3971
3972     if ($conf->{watchdog}) {
3973         my $wdopts = parse_watchdog($conf->{watchdog});
3974         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3975         my $watchdog = $wdopts->{model} || 'i6300esb';
3976         push @$devices, '-device', "$watchdog$pciaddr";
3977         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3978     }
3979
3980     my $vollist = [];
3981     my $scsicontroller = {};
3982     my $ahcicontroller = {};
3983     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3984
3985     # Add iscsi initiator name if available
3986     if (my $initiator = get_initiator_name()) {
3987         push @$devices, '-iscsi', "initiator-name=$initiator";
3988     }
3989
3990     foreach_drive($conf, sub {
3991         my ($ds, $drive) = @_;
3992
3993         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3994             push @$vollist, $drive->{file};
3995         }
3996
3997         # ignore efidisk here, already added in bios/fw handling code above
3998         return if $drive->{interface} eq 'efidisk';
3999
4000         $use_virtio = 1 if $ds =~ m/^virtio/;
4001
4002         if (drive_is_cdrom ($drive)) {
4003             if ($bootindex_hash->{d}) {
4004                 $drive->{bootindex} = $bootindex_hash->{d};
4005                 $bootindex_hash->{d} += 1;
4006             }
4007         } else {
4008             if ($bootindex_hash->{c}) {
4009                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4010                 $bootindex_hash->{c} += 1;
4011             }
4012         }
4013
4014         if($drive->{interface} eq 'virtio'){
4015            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4016         }
4017
4018         if ($drive->{interface} eq 'scsi') {
4019
4020             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4021
4022             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4023             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4024
4025             my $iothread = '';
4026             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4027                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4028                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4029             } elsif ($drive->{iothread}) {
4030                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4031             }
4032
4033             my $queues = '';
4034             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4035                 $queues = ",num_queues=$drive->{queues}";
4036             }
4037
4038             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4039             $scsicontroller->{$controller}=1;
4040         }
4041
4042         if ($drive->{interface} eq 'sata') {
4043            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4044            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4045            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4046            $ahcicontroller->{$controller}=1;
4047         }
4048
4049         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4050         push @$devices, '-drive',$drive_cmd;
4051         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4052     });
4053
4054     for (my $i = 0; $i < $MAX_NETS; $i++) {
4055          next if !$conf->{"net$i"};
4056          my $d = parse_net($conf->{"net$i"});
4057          next if !$d;
4058
4059          $use_virtio = 1 if $d->{model} eq 'virtio';
4060
4061          if ($bootindex_hash->{n}) {
4062             $d->{bootindex} = $bootindex_hash->{n};
4063             $bootindex_hash->{n} += 1;
4064          }
4065
4066          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4067          push @$devices, '-netdev', $netdevfull;
4068
4069          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4070          push @$devices, '-device', $netdevicefull;
4071     }
4072
4073     if ($conf->{ivshmem}) {
4074         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4075
4076         my $bus;
4077         if ($q35) {
4078             $bus = print_pcie_addr("ivshmem");
4079         } else {
4080             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4081         }
4082
4083         my $ivshmem_name = $ivshmem->{name} // $vmid;
4084         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4085
4086         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4087         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4088     }
4089
4090     if (!$q35) {
4091         # add pci bridges
4092         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4093            $bridges->{1} = 1;
4094            $bridges->{2} = 1;
4095         }
4096
4097         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4098
4099         for my $k (sort {$b cmp $a} keys %$bridges) {
4100             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4101             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4102         }
4103     }
4104
4105     push @$cmd, @$devices;
4106     push @$cmd, '-rtc', join(',', @$rtcFlags)
4107         if scalar(@$rtcFlags);
4108     push @$cmd, '-machine', join(',', @$machineFlags)
4109         if scalar(@$machineFlags);
4110     push @$cmd, '-global', join(',', @$globalFlags)
4111         if scalar(@$globalFlags);
4112
4113     if (my $vmstate = $conf->{vmstate}) {
4114         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4115         push @$vollist, $vmstate;
4116         push @$cmd, '-loadstate', $statepath;
4117     }
4118
4119     # add custom args
4120     if ($conf->{args}) {
4121         my $aa = PVE::Tools::split_args($conf->{args});
4122         push @$cmd, @$aa;
4123     }
4124
4125     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4126 }
4127
4128 sub vnc_socket {
4129     my ($vmid) = @_;
4130     return "${var_run_tmpdir}/$vmid.vnc";
4131 }
4132
4133 sub spice_port {
4134     my ($vmid) = @_;
4135
4136     my $res = vm_mon_cmd($vmid, 'query-spice');
4137
4138     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4139 }
4140
4141 sub qmp_socket {
4142     my ($vmid, $qga) = @_;
4143     my $sockettype = $qga ? 'qga' : 'qmp';
4144     return "${var_run_tmpdir}/$vmid.$sockettype";
4145 }
4146
4147 sub pidfile_name {
4148     my ($vmid) = @_;
4149     return "${var_run_tmpdir}/$vmid.pid";
4150 }
4151
4152 sub vm_devices_list {
4153     my ($vmid) = @_;
4154
4155     my $res = vm_mon_cmd($vmid, 'query-pci');
4156     my $devices_to_check = [];
4157     my $devices = {};
4158     foreach my $pcibus (@$res) {
4159         push @$devices_to_check, @{$pcibus->{devices}},
4160     }
4161
4162     while (@$devices_to_check) {
4163         my $to_check = [];
4164         for my $d (@$devices_to_check) {
4165             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4166             next if !$d->{'pci_bridge'};
4167
4168             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4169             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4170         }
4171         $devices_to_check = $to_check;
4172     }
4173
4174     my $resblock = vm_mon_cmd($vmid, 'query-block');
4175     foreach my $block (@$resblock) {
4176         if($block->{device} =~ m/^drive-(\S+)/){
4177                 $devices->{$1} = 1;
4178         }
4179     }
4180
4181     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4182     foreach my $mice (@$resmice) {
4183         if ($mice->{name} eq 'QEMU HID Tablet') {
4184             $devices->{tablet} = 1;
4185             last;
4186         }
4187     }
4188
4189     # for usb devices there is no query-usb
4190     # but we can iterate over the entries in
4191     # qom-list path=/machine/peripheral
4192     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4193     foreach my $per (@$resperipheral) {
4194         if ($per->{name} =~ m/^usb\d+$/) {
4195             $devices->{$per->{name}} = 1;
4196         }
4197     }
4198
4199     return $devices;
4200 }
4201
4202 sub vm_deviceplug {
4203     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4204
4205     my $q35 = machine_type_is_q35($conf);
4206
4207     my $devices_list = vm_devices_list($vmid);
4208     return 1 if defined($devices_list->{$deviceid});
4209
4210     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4211
4212     if ($deviceid eq 'tablet') {
4213
4214         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4215
4216     } elsif ($deviceid eq 'keyboard') {
4217
4218         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4219
4220     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4221
4222         die "usb hotplug currently not reliable\n";
4223         # since we can't reliably hot unplug all added usb devices
4224         # and usb passthrough disables live migration
4225         # we disable usb hotplugging for now
4226         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4227
4228     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4229
4230         qemu_iothread_add($vmid, $deviceid, $device);
4231
4232         qemu_driveadd($storecfg, $vmid, $device);
4233         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4234
4235         qemu_deviceadd($vmid, $devicefull);
4236         eval { qemu_deviceaddverify($vmid, $deviceid); };
4237         if (my $err = $@) {
4238             eval { qemu_drivedel($vmid, $deviceid); };
4239             warn $@ if $@;
4240             die $err;
4241         }
4242
4243     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4244
4245
4246         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4247         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4248         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4249
4250         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4251
4252         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4253             qemu_iothread_add($vmid, $deviceid, $device);
4254             $devicefull .= ",iothread=iothread-$deviceid";
4255         }
4256
4257         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4258             $devicefull .= ",num_queues=$device->{queues}";
4259         }
4260
4261         qemu_deviceadd($vmid, $devicefull);
4262         qemu_deviceaddverify($vmid, $deviceid);
4263
4264     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4265
4266         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4267         qemu_driveadd($storecfg, $vmid, $device);
4268
4269         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4270         eval { qemu_deviceadd($vmid, $devicefull); };
4271         if (my $err = $@) {
4272             eval { qemu_drivedel($vmid, $deviceid); };
4273             warn $@ if $@;
4274             die $err;
4275         }
4276
4277     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4278
4279         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4280
4281         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4282         my $use_old_bios_files = undef;
4283         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4284
4285         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4286         qemu_deviceadd($vmid, $netdevicefull);
4287         eval {
4288             qemu_deviceaddverify($vmid, $deviceid);
4289             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4290         };
4291         if (my $err = $@) {
4292             eval { qemu_netdevdel($vmid, $deviceid); };
4293             warn $@ if $@;
4294             die $err;
4295         }
4296
4297     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4298
4299         my $bridgeid = $2;
4300         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4301         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4302
4303         qemu_deviceadd($vmid, $devicefull);
4304         qemu_deviceaddverify($vmid, $deviceid);
4305
4306     } else {
4307         die "can't hotplug device '$deviceid'\n";
4308     }
4309
4310     return 1;
4311 }
4312
4313 # fixme: this should raise exceptions on error!
4314 sub vm_deviceunplug {
4315     my ($vmid, $conf, $deviceid) = @_;
4316
4317     my $devices_list = vm_devices_list($vmid);
4318     return 1 if !defined($devices_list->{$deviceid});
4319
4320     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4321
4322     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4323
4324         qemu_devicedel($vmid, $deviceid);
4325
4326     } elsif ($deviceid =~ m/^usb\d+$/) {
4327
4328         die "usb hotplug currently not reliable\n";
4329         # when unplugging usb devices this way,
4330         # there may be remaining usb controllers/hubs
4331         # so we disable it for now
4332         qemu_devicedel($vmid, $deviceid);
4333         qemu_devicedelverify($vmid, $deviceid);
4334
4335     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4336
4337         qemu_devicedel($vmid, $deviceid);
4338         qemu_devicedelverify($vmid, $deviceid);
4339         qemu_drivedel($vmid, $deviceid);
4340         qemu_iothread_del($conf, $vmid, $deviceid);
4341
4342     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4343
4344         qemu_devicedel($vmid, $deviceid);
4345         qemu_devicedelverify($vmid, $deviceid);
4346         qemu_iothread_del($conf, $vmid, $deviceid);
4347
4348     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4349
4350         qemu_devicedel($vmid, $deviceid);
4351         qemu_drivedel($vmid, $deviceid);
4352         qemu_deletescsihw($conf, $vmid, $deviceid);
4353
4354     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4355
4356         qemu_devicedel($vmid, $deviceid);
4357         qemu_devicedelverify($vmid, $deviceid);
4358         qemu_netdevdel($vmid, $deviceid);
4359
4360     } else {
4361         die "can't unplug device '$deviceid'\n";
4362     }
4363
4364     return 1;
4365 }
4366
4367 sub qemu_deviceadd {
4368     my ($vmid, $devicefull) = @_;
4369
4370     $devicefull = "driver=".$devicefull;
4371     my %options =  split(/[=,]/, $devicefull);
4372
4373     vm_mon_cmd($vmid, "device_add" , %options);
4374 }
4375
4376 sub qemu_devicedel {
4377     my ($vmid, $deviceid) = @_;
4378
4379     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4380 }
4381
4382 sub qemu_iothread_add {
4383     my($vmid, $deviceid, $device) = @_;
4384
4385     if ($device->{iothread}) {
4386         my $iothreads = vm_iothreads_list($vmid);
4387         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4388     }
4389 }
4390
4391 sub qemu_iothread_del {
4392     my($conf, $vmid, $deviceid) = @_;
4393
4394     my $confid = $deviceid;
4395     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4396         $confid = 'scsi' . $1;
4397     }
4398     my $device = parse_drive($confid, $conf->{$confid});
4399     if ($device->{iothread}) {
4400         my $iothreads = vm_iothreads_list($vmid);
4401         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4402     }
4403 }
4404
4405 sub qemu_objectadd {
4406     my($vmid, $objectid, $qomtype) = @_;
4407
4408     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4409
4410     return 1;
4411 }
4412
4413 sub qemu_objectdel {
4414     my($vmid, $objectid) = @_;
4415
4416     vm_mon_cmd($vmid, "object-del", id => $objectid);
4417
4418     return 1;
4419 }
4420
4421 sub qemu_driveadd {
4422     my ($storecfg, $vmid, $device) = @_;
4423
4424     my $drive = print_drive_full($storecfg, $vmid, $device);
4425     $drive =~ s/\\/\\\\/g;
4426     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4427
4428     # If the command succeeds qemu prints: "OK"
4429     return 1 if $ret =~ m/OK/s;
4430
4431     die "adding drive failed: $ret\n";
4432 }
4433
4434 sub qemu_drivedel {
4435     my($vmid, $deviceid) = @_;
4436
4437     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4438     $ret =~ s/^\s+//;
4439
4440     return 1 if $ret eq "";
4441
4442     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4443     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4444
4445     die "deleting drive $deviceid failed : $ret\n";
4446 }
4447
4448 sub qemu_deviceaddverify {
4449     my ($vmid, $deviceid) = @_;
4450
4451     for (my $i = 0; $i <= 5; $i++) {
4452          my $devices_list = vm_devices_list($vmid);
4453          return 1 if defined($devices_list->{$deviceid});
4454          sleep 1;
4455     }
4456
4457     die "error on hotplug device '$deviceid'\n";
4458 }
4459
4460
4461 sub qemu_devicedelverify {
4462     my ($vmid, $deviceid) = @_;
4463
4464     # need to verify that the device is correctly removed as device_del
4465     # is async and empty return is not reliable
4466
4467     for (my $i = 0; $i <= 5; $i++) {
4468          my $devices_list = vm_devices_list($vmid);
4469          return 1 if !defined($devices_list->{$deviceid});
4470          sleep 1;
4471     }
4472
4473     die "error on hot-unplugging device '$deviceid'\n";
4474 }
4475
4476 sub qemu_findorcreatescsihw {
4477     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4478
4479     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4480
4481     my $scsihwid="$controller_prefix$controller";
4482     my $devices_list = vm_devices_list($vmid);
4483
4484     if(!defined($devices_list->{$scsihwid})) {
4485         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4486     }
4487
4488     return 1;
4489 }
4490
4491 sub qemu_deletescsihw {
4492     my ($conf, $vmid, $opt) = @_;
4493
4494     my $device = parse_drive($opt, $conf->{$opt});
4495
4496     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4497         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4498         return 1;
4499     }
4500
4501     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4502
4503     my $devices_list = vm_devices_list($vmid);
4504     foreach my $opt (keys %{$devices_list}) {
4505         if (PVE::QemuServer::is_valid_drivename($opt)) {
4506             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4507             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4508                 return 1;
4509             }
4510         }
4511     }
4512
4513     my $scsihwid="scsihw$controller";
4514
4515     vm_deviceunplug($vmid, $conf, $scsihwid);
4516
4517     return 1;
4518 }
4519
4520 sub qemu_add_pci_bridge {
4521     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4522
4523     my $bridges = {};
4524
4525     my $bridgeid;
4526
4527     print_pci_addr($device, $bridges, $arch, $machine_type);
4528
4529     while (my ($k, $v) = each %$bridges) {
4530         $bridgeid = $k;
4531     }
4532     return 1 if !defined($bridgeid) || $bridgeid < 1;
4533
4534     my $bridge = "pci.$bridgeid";
4535     my $devices_list = vm_devices_list($vmid);
4536
4537     if (!defined($devices_list->{$bridge})) {
4538         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4539     }
4540
4541     return 1;
4542 }
4543
4544 sub qemu_set_link_status {
4545     my ($vmid, $device, $up) = @_;
4546
4547     vm_mon_cmd($vmid, "set_link", name => $device,
4548                up => $up ? JSON::true : JSON::false);
4549 }
4550
4551 sub qemu_netdevadd {
4552     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4553
4554     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4555     my %options =  split(/[=,]/, $netdev);
4556
4557     vm_mon_cmd($vmid, "netdev_add",  %options);
4558     return 1;
4559 }
4560
4561 sub qemu_netdevdel {
4562     my ($vmid, $deviceid) = @_;
4563
4564     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4565 }
4566
4567 sub qemu_usb_hotplug {
4568     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4569
4570     return if !$device;
4571
4572     # remove the old one first
4573     vm_deviceunplug($vmid, $conf, $deviceid);
4574
4575     # check if xhci controller is necessary and available
4576     if ($device->{usb3}) {
4577
4578         my $devicelist = vm_devices_list($vmid);
4579
4580         if (!$devicelist->{xhci}) {
4581             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4582             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4583         }
4584     }
4585     my $d = parse_usb_device($device->{host});
4586     $d->{usb3} = $device->{usb3};
4587
4588     # add the new one
4589     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4590 }
4591
4592 sub qemu_cpu_hotplug {
4593     my ($vmid, $conf, $vcpus) = @_;
4594
4595     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4596
4597     my $sockets = 1;
4598     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4599     $sockets = $conf->{sockets} if  $conf->{sockets};
4600     my $cores = $conf->{cores} || 1;
4601     my $maxcpus = $sockets * $cores;
4602
4603     $vcpus = $maxcpus if !$vcpus;
4604
4605     die "you can't add more vcpus than maxcpus\n"
4606         if $vcpus > $maxcpus;
4607
4608     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4609
4610     if ($vcpus < $currentvcpus) {
4611
4612         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4613
4614             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4615                 qemu_devicedel($vmid, "cpu$i");
4616                 my $retry = 0;
4617                 my $currentrunningvcpus = undef;
4618                 while (1) {
4619                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4620                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4621                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4622                     $retry++;
4623                     sleep 1;
4624                 }
4625                 #update conf after each succesfull cpu unplug
4626                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4627                 PVE::QemuConfig->write_config($vmid, $conf);
4628             }
4629         } else {
4630             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4631         }
4632
4633         return;
4634     }
4635
4636     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4637     die "vcpus in running vm does not match its configuration\n"
4638         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4639
4640     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4641
4642         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4643             my $cpustr = print_cpu_device($conf, $i);
4644             qemu_deviceadd($vmid, $cpustr);
4645
4646             my $retry = 0;
4647             my $currentrunningvcpus = undef;
4648             while (1) {
4649                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4650                 last if scalar(@{$currentrunningvcpus}) == $i;
4651                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4652                 sleep 1;
4653                 $retry++;
4654             }
4655             #update conf after each succesfull cpu hotplug
4656             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4657             PVE::QemuConfig->write_config($vmid, $conf);
4658         }
4659     } else {
4660
4661         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4662             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4663         }
4664     }
4665 }
4666
4667 sub qemu_block_set_io_throttle {
4668     my ($vmid, $deviceid,
4669         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4670         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4671         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4672         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4673
4674     return if !check_running($vmid) ;
4675
4676     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4677         bps => int($bps),
4678         bps_rd => int($bps_rd),
4679         bps_wr => int($bps_wr),
4680         iops => int($iops),
4681         iops_rd => int($iops_rd),
4682         iops_wr => int($iops_wr),
4683         bps_max => int($bps_max),
4684         bps_rd_max => int($bps_rd_max),
4685         bps_wr_max => int($bps_wr_max),
4686         iops_max => int($iops_max),
4687         iops_rd_max => int($iops_rd_max),
4688         iops_wr_max => int($iops_wr_max),
4689         bps_max_length => int($bps_max_length),
4690         bps_rd_max_length => int($bps_rd_max_length),
4691         bps_wr_max_length => int($bps_wr_max_length),
4692         iops_max_length => int($iops_max_length),
4693         iops_rd_max_length => int($iops_rd_max_length),
4694         iops_wr_max_length => int($iops_wr_max_length),
4695     );
4696
4697 }
4698
4699 # old code, only used to shutdown old VM after update
4700 sub __read_avail {
4701     my ($fh, $timeout) = @_;
4702
4703     my $sel = new IO::Select;
4704     $sel->add($fh);
4705
4706     my $res = '';
4707     my $buf;
4708
4709     my @ready;
4710     while (scalar (@ready = $sel->can_read($timeout))) {
4711         my $count;
4712         if ($count = $fh->sysread($buf, 8192)) {
4713             if ($buf =~ /^(.*)\(qemu\) $/s) {
4714                 $res .= $1;
4715                 last;
4716             } else {
4717                 $res .= $buf;
4718             }
4719         } else {
4720             if (!defined($count)) {
4721                 die "$!\n";
4722             }
4723             last;
4724         }
4725     }
4726
4727     die "monitor read timeout\n" if !scalar(@ready);
4728
4729     return $res;
4730 }
4731
4732 sub qemu_block_resize {
4733     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4734
4735     my $running = check_running($vmid);
4736
4737     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4738
4739     return if !$running;
4740
4741     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4742
4743 }
4744
4745 sub qemu_volume_snapshot {
4746     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4747
4748     my $running = check_running($vmid);
4749
4750     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4751         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4752     } else {
4753         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4754     }
4755 }
4756
4757 sub qemu_volume_snapshot_delete {
4758     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4759
4760     my $running = check_running($vmid);
4761
4762     if($running) {
4763
4764         $running = undef;
4765         my $conf = PVE::QemuConfig->load_config($vmid);
4766         foreach_drive($conf, sub {
4767             my ($ds, $drive) = @_;
4768             $running = 1 if $drive->{file} eq $volid;
4769         });
4770     }
4771
4772     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4773         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4774     } else {
4775         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4776     }
4777 }
4778
4779 sub set_migration_caps {
4780     my ($vmid) = @_;
4781
4782     my $cap_ref = [];
4783
4784     my $enabled_cap = {
4785         "auto-converge" => 1,
4786         "xbzrle" => 1,
4787         "x-rdma-pin-all" => 0,
4788         "zero-blocks" => 0,
4789         "compress" => 0
4790     };
4791
4792     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4793
4794     for my $supported_capability (@$supported_capabilities) {
4795         push @$cap_ref, {
4796             capability => $supported_capability->{capability},
4797             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4798         };
4799     }
4800
4801     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4802 }
4803
4804 my $fast_plug_option = {
4805     'lock' => 1,
4806     'name' => 1,
4807     'onboot' => 1,
4808     'shares' => 1,
4809     'startup' => 1,
4810     'description' => 1,
4811     'protection' => 1,
4812     'vmstatestorage' => 1,
4813     'hookscript' => 1,
4814 };
4815
4816 # hotplug changes in [PENDING]
4817 # $selection hash can be used to only apply specified options, for
4818 # example: { cores => 1 } (only apply changed 'cores')
4819 # $errors ref is used to return error messages
4820 sub vmconfig_hotplug_pending {
4821     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4822
4823     my $defaults = load_defaults();
4824     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4825
4826     # commit values which do not have any impact on running VM first
4827     # Note: those option cannot raise errors, we we do not care about
4828     # $selection and always apply them.
4829
4830     my $add_error = sub {
4831         my ($opt, $msg) = @_;
4832         $errors->{$opt} = "hotplug problem - $msg";
4833     };
4834
4835     my $changes = 0;
4836     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4837         if ($fast_plug_option->{$opt}) {
4838             $conf->{$opt} = $conf->{pending}->{$opt};
4839             delete $conf->{pending}->{$opt};
4840             $changes = 1;
4841         }
4842     }
4843
4844     if ($changes) {
4845         PVE::QemuConfig->write_config($vmid, $conf);
4846         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4847     }
4848
4849     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4850
4851     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4852     foreach my $opt (sort keys %$pending_delete_hash) {
4853         next if $selection && !$selection->{$opt};
4854         my $force = $pending_delete_hash->{$opt}->{force};
4855         eval {
4856             if ($opt eq 'hotplug') {
4857                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4858             } elsif ($opt eq 'tablet') {
4859                 die "skip\n" if !$hotplug_features->{usb};
4860                 if ($defaults->{tablet}) {
4861                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4862                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4863                         if $arch eq 'aarch64';
4864                 } else {
4865                     vm_deviceunplug($vmid, $conf, 'tablet');
4866                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4867                 }
4868             } elsif ($opt =~ m/^usb\d+/) {
4869                 die "skip\n";
4870                 # since we cannot reliably hot unplug usb devices
4871                 # we are disabling it
4872                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4873                 vm_deviceunplug($vmid, $conf, $opt);
4874             } elsif ($opt eq 'vcpus') {
4875                 die "skip\n" if !$hotplug_features->{cpu};
4876                 qemu_cpu_hotplug($vmid, $conf, undef);
4877             } elsif ($opt eq 'balloon') {
4878                 # enable balloon device is not hotpluggable
4879                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4880                 # here we reset the ballooning value to memory
4881                 my $balloon = $conf->{memory} || $defaults->{memory};
4882                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4883             } elsif ($fast_plug_option->{$opt}) {
4884                 # do nothing
4885             } elsif ($opt =~ m/^net(\d+)$/) {
4886                 die "skip\n" if !$hotplug_features->{network};
4887                 vm_deviceunplug($vmid, $conf, $opt);
4888             } elsif (is_valid_drivename($opt)) {
4889                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4890                 vm_deviceunplug($vmid, $conf, $opt);
4891                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4892             } elsif ($opt =~ m/^memory$/) {
4893                 die "skip\n" if !$hotplug_features->{memory};
4894                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4895             } elsif ($opt eq 'cpuunits') {
4896                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4897             } elsif ($opt eq 'cpulimit') {
4898                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4899             } else {
4900                 die "skip\n";
4901             }
4902         };
4903         if (my $err = $@) {
4904             &$add_error($opt, $err) if $err ne "skip\n";
4905         } else {
4906             # save new config if hotplug was successful
4907             delete $conf->{$opt};
4908             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4909             PVE::QemuConfig->write_config($vmid, $conf);
4910             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4911         }
4912     }
4913
4914     my $apply_pending_cloudinit;
4915     $apply_pending_cloudinit = sub {
4916         my ($key, $value) = @_;
4917         $apply_pending_cloudinit = sub {}; # once is enough
4918
4919         my @cloudinit_opts = keys %$confdesc_cloudinit;
4920         foreach my $opt (keys %{$conf->{pending}}) {
4921             next if !grep { $_ eq $opt } @cloudinit_opts;
4922             $conf->{$opt} = delete $conf->{pending}->{$opt};
4923         }
4924
4925         my $new_conf = { %$conf };
4926         $new_conf->{$key} = $value;
4927         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4928     };
4929
4930     foreach my $opt (keys %{$conf->{pending}}) {
4931         next if $selection && !$selection->{$opt};
4932         my $value = $conf->{pending}->{$opt};
4933         eval {
4934             if ($opt eq 'hotplug') {
4935                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4936             } elsif ($opt eq 'tablet') {
4937                 die "skip\n" if !$hotplug_features->{usb};
4938                 if ($value == 1) {
4939                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4940                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4941                         if $arch eq 'aarch64';
4942                 } elsif ($value == 0) {
4943                     vm_deviceunplug($vmid, $conf, 'tablet');
4944                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4945                 }
4946             } elsif ($opt =~ m/^usb\d+$/) {
4947                 die "skip\n";
4948                 # since we cannot reliably hot unplug usb devices
4949                 # we are disabling it
4950                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4951                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4952                 die "skip\n" if !$d;
4953                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4954             } elsif ($opt eq 'vcpus') {
4955                 die "skip\n" if !$hotplug_features->{cpu};
4956                 qemu_cpu_hotplug($vmid, $conf, $value);
4957             } elsif ($opt eq 'balloon') {
4958                 # enable/disable balloning device is not hotpluggable
4959                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4960                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4961                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4962
4963                 # allow manual ballooning if shares is set to zero
4964                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4965                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4966                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4967                 }
4968             } elsif ($opt =~ m/^net(\d+)$/) {
4969                 # some changes can be done without hotplug
4970                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4971                                     $vmid, $opt, $value, $arch, $machine_type);
4972             } elsif (is_valid_drivename($opt)) {
4973                 # some changes can be done without hotplug
4974                 my $drive = parse_drive($opt, $value);
4975                 if (drive_is_cloudinit($drive)) {
4976                     &$apply_pending_cloudinit($opt, $value);
4977                 }
4978                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4979                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4980             } elsif ($opt =~ m/^memory$/) { #dimms
4981                 die "skip\n" if !$hotplug_features->{memory};
4982                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4983             } elsif ($opt eq 'cpuunits') {
4984                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4985             } elsif ($opt eq 'cpulimit') {
4986                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4987                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4988             } else {
4989                 die "skip\n";  # skip non-hot-pluggable options
4990             }
4991         };
4992         if (my $err = $@) {
4993             &$add_error($opt, $err) if $err ne "skip\n";
4994         } else {
4995             # save new config if hotplug was successful
4996             $conf->{$opt} = $value;
4997             delete $conf->{pending}->{$opt};
4998             PVE::QemuConfig->write_config($vmid, $conf);
4999             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5000         }
5001     }
5002 }
5003
5004 sub try_deallocate_drive {
5005     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5006
5007     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5008         my $volid = $drive->{file};
5009         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5010             my $sid = PVE::Storage::parse_volume_id($volid);
5011             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5012
5013             # check if the disk is really unused
5014             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5015                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5016             PVE::Storage::vdisk_free($storecfg, $volid);
5017             return 1;
5018         } else {
5019             # If vm is not owner of this disk remove from config
5020             return 1;
5021         }
5022     }
5023
5024     return undef;
5025 }
5026
5027 sub vmconfig_delete_or_detach_drive {
5028     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5029
5030     my $drive = parse_drive($opt, $conf->{$opt});
5031
5032     my $rpcenv = PVE::RPCEnvironment::get();
5033     my $authuser = $rpcenv->get_user();
5034
5035     if ($force) {
5036         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5037         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5038     } else {
5039         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5040     }
5041 }
5042
5043
5044
5045 sub vmconfig_apply_pending {
5046     my ($vmid, $conf, $storecfg) = @_;
5047
5048     # cold plug
5049
5050     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5051     foreach my $opt (sort keys %$pending_delete_hash) {
5052         die "internal error" if $opt =~ m/^unused/;
5053         my $force = $pending_delete_hash->{$opt}->{force};
5054         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5055         if (!defined($conf->{$opt})) {
5056             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5057             PVE::QemuConfig->write_config($vmid, $conf);
5058         } elsif (is_valid_drivename($opt)) {
5059             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5060             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5061             delete $conf->{$opt};
5062             PVE::QemuConfig->write_config($vmid, $conf);
5063         } else {
5064             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5065             delete $conf->{$opt};
5066             PVE::QemuConfig->write_config($vmid, $conf);
5067         }
5068     }
5069
5070     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5071
5072     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5073         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5074
5075         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5076             # skip if nothing changed
5077         } elsif (is_valid_drivename($opt)) {
5078             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5079                 if defined($conf->{$opt});
5080             $conf->{$opt} = $conf->{pending}->{$opt};
5081         } else {
5082             $conf->{$opt} = $conf->{pending}->{$opt};
5083         }
5084
5085         delete $conf->{pending}->{$opt};
5086         PVE::QemuConfig->write_config($vmid, $conf);
5087     }
5088 }
5089
5090 my $safe_num_ne = sub {
5091     my ($a, $b) = @_;
5092
5093     return 0 if !defined($a) && !defined($b);
5094     return 1 if !defined($a);
5095     return 1 if !defined($b);
5096
5097     return $a != $b;
5098 };
5099
5100 my $safe_string_ne = sub {
5101     my ($a, $b) = @_;
5102
5103     return 0 if !defined($a) && !defined($b);
5104     return 1 if !defined($a);
5105     return 1 if !defined($b);
5106
5107     return $a ne $b;
5108 };
5109
5110 sub vmconfig_update_net {
5111     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5112
5113     my $newnet = parse_net($value);
5114
5115     if ($conf->{$opt}) {
5116         my $oldnet = parse_net($conf->{$opt});
5117
5118         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5119             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5120             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5121             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5122
5123             # for non online change, we try to hot-unplug
5124             die "skip\n" if !$hotplug;
5125             vm_deviceunplug($vmid, $conf, $opt);
5126         } else {
5127
5128             die "internal error" if $opt !~ m/net(\d+)/;
5129             my $iface = "tap${vmid}i$1";
5130
5131             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5132                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5133                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5134                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5135                 PVE::Network::tap_unplug($iface);
5136                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5137             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5138                 # Rate can be applied on its own but any change above needs to
5139                 # include the rate in tap_plug since OVS resets everything.
5140                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5141             }
5142
5143             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5144                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5145             }
5146
5147             return 1;
5148         }
5149     }
5150
5151     if ($hotplug) {
5152         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5153     } else {
5154         die "skip\n";
5155     }
5156 }
5157
5158 sub vmconfig_update_disk {
5159     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5160
5161     # fixme: do we need force?
5162
5163     my $drive = parse_drive($opt, $value);
5164
5165     if ($conf->{$opt}) {
5166
5167         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5168
5169             my $media = $drive->{media} || 'disk';
5170             my $oldmedia = $old_drive->{media} || 'disk';
5171             die "unable to change media type\n" if $media ne $oldmedia;
5172
5173             if (!drive_is_cdrom($old_drive)) {
5174
5175                 if ($drive->{file} ne $old_drive->{file}) {
5176
5177                     die "skip\n" if !$hotplug;
5178
5179                     # unplug and register as unused
5180                     vm_deviceunplug($vmid, $conf, $opt);
5181                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5182
5183                 } else {
5184                     # update existing disk
5185
5186                     # skip non hotpluggable value
5187                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5188                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5189                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5190                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5191                         die "skip\n";
5192                     }
5193
5194                     # apply throttle
5195                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5196                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5197                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5198                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5199                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5200                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5201                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5202                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5203                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5204                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5205                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5206                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5207                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5208                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5209                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5210                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5211                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5212                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5213
5214                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5215                                                    ($drive->{mbps} || 0)*1024*1024,
5216                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5217                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5218                                                    $drive->{iops} || 0,
5219                                                    $drive->{iops_rd} || 0,
5220                                                    $drive->{iops_wr} || 0,
5221                                                    ($drive->{mbps_max} || 0)*1024*1024,
5222                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5223                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5224                                                    $drive->{iops_max} || 0,
5225                                                    $drive->{iops_rd_max} || 0,
5226                                                    $drive->{iops_wr_max} || 0,
5227                                                    $drive->{bps_max_length} || 1,
5228                                                    $drive->{bps_rd_max_length} || 1,
5229                                                    $drive->{bps_wr_max_length} || 1,
5230                                                    $drive->{iops_max_length} || 1,
5231                                                    $drive->{iops_rd_max_length} || 1,
5232                                                    $drive->{iops_wr_max_length} || 1);
5233
5234                     }
5235
5236                     return 1;
5237                 }
5238
5239             } else { # cdrom
5240
5241                 if ($drive->{file} eq 'none') {
5242                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5243                     if (drive_is_cloudinit($old_drive)) {
5244                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5245                     }
5246                 } else {
5247                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5248                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5249                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5250                 }
5251
5252                 return 1;
5253             }
5254         }
5255     }
5256
5257     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5258     # hotplug new disks
5259     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5260     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5261 }
5262
5263 sub vm_start {
5264     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5265         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5266
5267     PVE::QemuConfig->lock_config($vmid, sub {
5268         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5269
5270         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5271
5272         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5273
5274         PVE::QemuConfig->check_lock($conf)
5275             if !($skiplock || $is_suspended);
5276
5277         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5278
5279         # clean up leftover reboot request files
5280         eval { clear_reboot_request($vmid); };
5281         warn $@ if $@;
5282
5283         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5284             vmconfig_apply_pending($vmid, $conf, $storecfg);
5285             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5286         }
5287
5288         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5289
5290         my $defaults = load_defaults();
5291
5292         # set environment variable useful inside network script
5293         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5294
5295         my $local_volumes = {};
5296
5297         if ($targetstorage) {
5298             foreach_drive($conf, sub {
5299                 my ($ds, $drive) = @_;
5300
5301                 return if drive_is_cdrom($drive);
5302
5303                 my $volid = $drive->{file};
5304
5305                 return if !$volid;
5306
5307                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5308
5309                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5310                 return if $scfg->{shared};
5311                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5312             });
5313
5314             my $format = undef;
5315
5316             foreach my $opt (sort keys %$local_volumes) {
5317
5318                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5319                 my $drive = parse_drive($opt, $conf->{$opt});
5320
5321                 #if remote storage is specified, use default format
5322                 if ($targetstorage && $targetstorage ne "1") {
5323                     $storeid = $targetstorage;
5324                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5325                     $format = $defFormat;
5326                 } else {
5327                     #else we use same format than original
5328                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5329                     $format = qemu_img_format($scfg, $volid);
5330                 }
5331
5332                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5333                 my $newdrive = $drive;
5334                 $newdrive->{format} = $format;
5335                 $newdrive->{file} = $newvolid;
5336                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5337                 $local_volumes->{$opt} = $drivestr;
5338                 #pass drive to conf for command line
5339                 $conf->{$opt} = $drivestr;
5340             }
5341         }
5342
5343         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5344
5345         if ($is_suspended) {
5346             # enforce machine type on suspended vm to ensure HW compatibility
5347             $forcemachine = $conf->{runningmachine};
5348             print "Resuming suspended VM\n";
5349         }
5350
5351         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5352
5353         my $migrate_uri;
5354         if ($statefile) {
5355             if ($statefile eq 'tcp') {
5356                 my $localip = "localhost";
5357                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5358                 my $nodename = PVE::INotify::nodename();
5359
5360                 if (!defined($migration_type)) {
5361                     if (defined($datacenterconf->{migration}->{type})) {
5362                         $migration_type = $datacenterconf->{migration}->{type};
5363                     } else {
5364                         $migration_type = 'secure';
5365                     }
5366                 }
5367
5368                 if ($migration_type eq 'insecure') {
5369                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5370                     if ($migrate_network_addr) {
5371                         $localip = $migrate_network_addr;
5372                     } else {
5373                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5374                     }
5375
5376                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5377                 }
5378
5379                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5380                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5381                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5382                 push @$cmd, '-incoming', $migrate_uri;
5383                 push @$cmd, '-S';
5384
5385             } elsif ($statefile eq 'unix') {
5386                 # should be default for secure migrations as a ssh TCP forward
5387                 # tunnel is not deterministic reliable ready and fails regurarly
5388                 # to set up in time, so use UNIX socket forwards
5389                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5390                 unlink $socket_addr;
5391
5392                 $migrate_uri = "unix:$socket_addr";
5393
5394                 push @$cmd, '-incoming', $migrate_uri;
5395                 push @$cmd, '-S';
5396
5397             } elsif (-e $statefile) {
5398                 push @$cmd, '-loadstate', $statefile;
5399             } else {
5400                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5401                 push @$vollist, $statefile;
5402                 push @$cmd, '-loadstate', $statepath;
5403             }
5404         } elsif ($paused) {
5405             push @$cmd, '-S';
5406         }
5407
5408         # host pci devices
5409         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5410           my $d = parse_hostpci($conf->{"hostpci$i"});
5411           next if !$d;
5412           my $pcidevices = $d->{pciid};
5413           foreach my $pcidevice (@$pcidevices) {
5414                 my $pciid = $pcidevice->{id};
5415
5416                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5417                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5418                 die "no pci device info for device '$pciid'\n" if !$info;
5419
5420                 if ($d->{mdev}) {
5421                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5422                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5423                 } else {
5424                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5425                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5426                     die "can't reset pci device '$pciid'\n"
5427                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5428                 }
5429           }
5430         }
5431
5432         PVE::Storage::activate_volumes($storecfg, $vollist);
5433
5434         eval {
5435             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5436                 outfunc => sub {}, errfunc => sub {});
5437         };
5438         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5439         # timeout should be more than enough here...
5440         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5441
5442         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5443                                                   : $defaults->{cpuunits};
5444
5445         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5446         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5447
5448         my %properties = (
5449             Slice => 'qemu.slice',
5450             KillMode => 'none',
5451             CPUShares => $cpuunits
5452         );
5453
5454         if (my $cpulimit = $conf->{cpulimit}) {
5455             $properties{CPUQuota} = int($cpulimit * 100);
5456         }
5457         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5458
5459         my $run_qemu = sub {
5460             PVE::Tools::run_fork sub {
5461                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5462                 run_command($cmd, %run_params);
5463             };
5464         };
5465
5466         if ($conf->{hugepages}) {
5467
5468             my $code = sub {
5469                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5470                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5471
5472                 PVE::QemuServer::Memory::hugepages_mount();
5473                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5474
5475                 eval { $run_qemu->() };
5476                 if (my $err = $@) {
5477                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5478                     die $err;
5479                 }
5480
5481                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5482             };
5483             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5484
5485         } else {
5486             eval { $run_qemu->() };
5487         }
5488
5489         if (my $err = $@) {
5490             # deactivate volumes if start fails
5491             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5492             die "start failed: $err";
5493         }
5494
5495         print "migration listens on $migrate_uri\n" if $migrate_uri;
5496
5497         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5498             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5499             warn $@ if $@;
5500         }
5501
5502         #start nbd server for storage migration
5503         if ($targetstorage) {
5504             my $nodename = PVE::INotify::nodename();
5505             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5506             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5507             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5508             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5509
5510             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5511
5512             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5513
5514             foreach my $opt (sort keys %$local_volumes) {
5515                 my $volid = $local_volumes->{$opt};
5516                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5517                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5518                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5519             }
5520         }
5521
5522         if ($migratedfrom) {
5523             eval {
5524                 set_migration_caps($vmid);
5525             };
5526             warn $@ if $@;
5527
5528             if ($spice_port) {
5529                 print "spice listens on port $spice_port\n";
5530                 if ($spice_ticket) {
5531                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5532                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5533                 }
5534             }
5535
5536         } else {
5537             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5538                 if !$statefile && $conf->{balloon};
5539
5540             foreach my $opt (keys %$conf) {
5541                 next if $opt !~  m/^net\d+$/;
5542                 my $nicconf = parse_net($conf->{$opt});
5543                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5544             }
5545         }
5546
5547         vm_mon_cmd_nocheck($vmid, 'qom-set',
5548                     path => "machine/peripheral/balloon0",
5549                     property => "guest-stats-polling-interval",
5550                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5551
5552         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5553             print "Resumed VM, removing state\n";
5554             delete $conf->@{qw(lock vmstate runningmachine)};
5555             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5556             PVE::Storage::vdisk_free($storecfg, $vmstate);
5557             PVE::QemuConfig->write_config($vmid, $conf);
5558         }
5559
5560         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5561     });
5562 }
5563
5564 sub vm_mon_cmd {
5565     my ($vmid, $execute, %params) = @_;
5566
5567     my $cmd = { execute => $execute, arguments => \%params };
5568     vm_qmp_command($vmid, $cmd);
5569 }
5570
5571 sub vm_mon_cmd_nocheck {
5572     my ($vmid, $execute, %params) = @_;
5573
5574     my $cmd = { execute => $execute, arguments => \%params };
5575     vm_qmp_command($vmid, $cmd, 1);
5576 }
5577
5578 sub vm_qmp_command {
5579     my ($vmid, $cmd, $nocheck) = @_;
5580
5581     my $res;
5582
5583     my $timeout;
5584     if ($cmd->{arguments}) {
5585         $timeout = delete $cmd->{arguments}->{timeout};
5586     }
5587
5588     eval {
5589         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5590         my $sname = qmp_socket($vmid);
5591         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5592             my $qmpclient = PVE::QMPClient->new();
5593
5594             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5595         } else {
5596             die "unable to open monitor socket\n";
5597         }
5598     };
5599     if (my $err = $@) {
5600         syslog("err", "VM $vmid qmp command failed - $err");
5601         die $err;
5602     }
5603
5604     return $res;
5605 }
5606
5607 sub vm_human_monitor_command {
5608     my ($vmid, $cmdline) = @_;
5609
5610     my $cmd = {
5611         execute => 'human-monitor-command',
5612         arguments => { 'command-line' => $cmdline},
5613     };
5614
5615     return vm_qmp_command($vmid, $cmd);
5616 }
5617
5618 sub vm_commandline {
5619     my ($storecfg, $vmid, $snapname) = @_;
5620
5621     my $conf = PVE::QemuConfig->load_config($vmid);
5622
5623     if ($snapname) {
5624         my $snapshot = $conf->{snapshots}->{$snapname};
5625         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5626
5627         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5628
5629         $conf = $snapshot;
5630     }
5631
5632     my $defaults = load_defaults();
5633
5634     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5635
5636     return PVE::Tools::cmd2string($cmd);
5637 }
5638
5639 sub vm_reset {
5640     my ($vmid, $skiplock) = @_;
5641
5642     PVE::QemuConfig->lock_config($vmid, sub {
5643
5644         my $conf = PVE::QemuConfig->load_config($vmid);
5645
5646         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5647
5648         vm_mon_cmd($vmid, "system_reset");
5649     });
5650 }
5651
5652 sub get_vm_volumes {
5653     my ($conf) = @_;
5654
5655     my $vollist = [];
5656     foreach_volid($conf, sub {
5657         my ($volid, $attr) = @_;
5658
5659         return if $volid =~ m|^/|;
5660
5661         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5662         return if !$sid;
5663
5664         push @$vollist, $volid;
5665     });
5666
5667     return $vollist;
5668 }
5669
5670 sub vm_stop_cleanup {
5671     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5672
5673     eval {
5674
5675         if (!$keepActive) {
5676             my $vollist = get_vm_volumes($conf);
5677             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5678         }
5679
5680         foreach my $ext (qw(mon qmp pid vnc qga)) {
5681             unlink "/var/run/qemu-server/${vmid}.$ext";
5682         }
5683
5684         if ($conf->{ivshmem}) {
5685             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5686             # just delete it for now, VMs which have this already open do not
5687             # are affected, but new VMs will get a separated one. If this
5688             # becomes an issue we either add some sort of ref-counting or just
5689             # add a "don't delete on stop" flag to the ivshmem format.
5690             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5691         }
5692
5693         foreach my $key (keys %$conf) {
5694             next if $key !~ m/^hostpci(\d+)$/;
5695             my $hostpciindex = $1;
5696             my $d = parse_hostpci($conf->{$key});
5697             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5698
5699             foreach my $pci (@{$d->{pciid}}) {
5700                 my $pciid = $pci->{id};
5701                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5702             }
5703         }
5704
5705         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5706     };
5707     warn $@ if $@; # avoid errors - just warn
5708 }
5709
5710 # call only in locked context
5711 sub _do_vm_stop {
5712     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5713
5714     my $pid = check_running($vmid, $nocheck);
5715     return if !$pid;
5716
5717     my $conf;
5718     if (!$nocheck) {
5719         $conf = PVE::QemuConfig->load_config($vmid);
5720         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5721         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5722             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5723             $timeout = $opts->{down} if $opts->{down};
5724         }
5725         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5726     }
5727
5728     eval {
5729         if ($shutdown) {
5730             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5731                 vm_qmp_command($vmid, {
5732                         execute => "guest-shutdown",
5733                         arguments => { timeout => $timeout }
5734                     }, $nocheck);
5735             } else {
5736                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5737             }
5738         } else {
5739             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5740         }
5741     };
5742     my $err = $@;
5743
5744     if (!$err) {
5745         $timeout = 60 if !defined($timeout);
5746
5747         my $count = 0;
5748         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5749             $count++;
5750             sleep 1;
5751         }
5752
5753         if ($count >= $timeout) {
5754             if ($force) {
5755                 warn "VM still running - terminating now with SIGTERM\n";
5756                 kill 15, $pid;
5757             } else {
5758                 die "VM quit/powerdown failed - got timeout\n";
5759             }
5760         } else {
5761             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5762             return;
5763         }
5764     } else {
5765         if ($force) {
5766             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5767             kill 15, $pid;
5768         } else {
5769             die "VM quit/powerdown failed\n";
5770         }
5771     }
5772
5773     # wait again
5774     $timeout = 10;
5775
5776     my $count = 0;
5777     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5778         $count++;
5779         sleep 1;
5780     }
5781
5782     if ($count >= $timeout) {
5783         warn "VM still running - terminating now with SIGKILL\n";
5784         kill 9, $pid;
5785         sleep 1;
5786     }
5787
5788     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5789 }
5790
5791 # Note: use $nocheck to skip tests if VM configuration file exists.
5792 # We need that when migration VMs to other nodes (files already moved)
5793 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5794 sub vm_stop {
5795     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5796
5797     $force = 1 if !defined($force) && !$shutdown;
5798
5799     if ($migratedfrom){
5800         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5801         kill 15, $pid if $pid;
5802         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5803         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5804         return;
5805     }
5806
5807     PVE::QemuConfig->lock_config($vmid, sub {
5808         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5809    });
5810 }
5811
5812 sub vm_reboot {
5813     my ($vmid, $timeout) = @_;
5814
5815     PVE::QemuConfig->lock_config($vmid, sub {
5816
5817         # only reboot if running, as qmeventd starts it again on a stop event
5818         return if !check_running($vmid);
5819
5820         create_reboot_request($vmid);
5821
5822         my $storecfg = PVE::Storage::config();
5823         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5824
5825    });
5826 }
5827
5828 sub vm_suspend {
5829     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5830
5831     my $conf;
5832     my $path;
5833     my $storecfg;
5834     my $vmstate;
5835
5836     PVE::QemuConfig->lock_config($vmid, sub {
5837
5838         $conf = PVE::QemuConfig->load_config($vmid);
5839
5840         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5841         PVE::QemuConfig->check_lock($conf)
5842             if !($skiplock || $is_backing_up);
5843
5844         die "cannot suspend to disk during backup\n"
5845             if $is_backing_up && $includestate;
5846
5847         if ($includestate) {
5848             $conf->{lock} = 'suspending';
5849             my $date = strftime("%Y-%m-%d", localtime(time()));
5850             $storecfg = PVE::Storage::config();
5851             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5852             $path = PVE::Storage::path($storecfg, $vmstate);
5853             PVE::QemuConfig->write_config($vmid, $conf);
5854         } else {
5855             vm_mon_cmd($vmid, "stop");
5856         }
5857     });
5858
5859     if ($includestate) {
5860         # save vm state
5861         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5862
5863         eval {
5864             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5865             for(;;) {
5866                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5867                 if (!$state->{status}) {
5868                     die "savevm not active\n";
5869                 } elsif ($state->{status} eq 'active') {
5870                     sleep(1);
5871                     next;
5872                 } elsif ($state->{status} eq 'completed') {
5873                     print "State saved, quitting\n";
5874                     last;
5875                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5876                     die "query-savevm failed with error '$state->{error}'\n"
5877                 } else {
5878                     die "query-savevm returned status '$state->{status}'\n";
5879                 }
5880             }
5881         };
5882         my $err = $@;
5883
5884         PVE::QemuConfig->lock_config($vmid, sub {
5885             $conf = PVE::QemuConfig->load_config($vmid);
5886             if ($err) {
5887                 # cleanup, but leave suspending lock, to indicate something went wrong
5888                 eval {
5889                     vm_mon_cmd($vmid, "savevm-end");
5890                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5891                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5892                     delete $conf->@{qw(vmstate runningmachine)};
5893                     PVE::QemuConfig->write_config($vmid, $conf);
5894                 };
5895                 warn $@ if $@;
5896                 die $err;
5897             }
5898
5899             die "lock changed unexpectedly\n"
5900                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5901
5902             vm_qmp_command($vmid, { execute => "quit" });
5903             $conf->{lock} = 'suspended';
5904             PVE::QemuConfig->write_config($vmid, $conf);
5905         });
5906     }
5907 }
5908
5909 sub vm_resume {
5910     my ($vmid, $skiplock, $nocheck) = @_;
5911
5912     PVE::QemuConfig->lock_config($vmid, sub {
5913         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5914         my $res = $vm_mon_cmd->($vmid, 'query-status');
5915         my $resume_cmd = 'cont';
5916
5917         if ($res->{status} && $res->{status} eq 'suspended') {
5918             $resume_cmd = 'system_wakeup';
5919         }
5920
5921         if (!$nocheck) {
5922
5923             my $conf = PVE::QemuConfig->load_config($vmid);
5924
5925             PVE::QemuConfig->check_lock($conf)
5926                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5927         }
5928
5929         $vm_mon_cmd->($vmid, $resume_cmd);
5930     });
5931 }
5932
5933 sub vm_sendkey {
5934     my ($vmid, $skiplock, $key) = @_;
5935
5936     PVE::QemuConfig->lock_config($vmid, sub {
5937
5938         my $conf = PVE::QemuConfig->load_config($vmid);
5939
5940         # there is no qmp command, so we use the human monitor command
5941         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5942         die $res if $res ne '';
5943     });
5944 }
5945
5946 sub vm_destroy {
5947     my ($storecfg, $vmid, $skiplock) = @_;
5948
5949     PVE::QemuConfig->lock_config($vmid, sub {
5950
5951         my $conf = PVE::QemuConfig->load_config($vmid);
5952
5953         if (!check_running($vmid)) {
5954             destroy_vm($storecfg, $vmid, undef, $skiplock);
5955         } else {
5956             die "VM $vmid is running - destroy failed\n";
5957         }
5958     });
5959 }
5960
5961 # vzdump restore implementaion
5962
5963 sub tar_archive_read_firstfile {
5964     my $archive = shift;
5965
5966     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5967
5968     # try to detect archive type first
5969     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5970         die "unable to open file '$archive'\n";
5971     my $firstfile = <$fh>;
5972     kill 15, $pid;
5973     close $fh;
5974
5975     die "ERROR: archive contaions no data\n" if !$firstfile;
5976     chomp $firstfile;
5977
5978     return $firstfile;
5979 }
5980
5981 sub tar_restore_cleanup {
5982     my ($storecfg, $statfile) = @_;
5983
5984     print STDERR "starting cleanup\n";
5985
5986     if (my $fd = IO::File->new($statfile, "r")) {
5987         while (defined(my $line = <$fd>)) {
5988             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5989                 my $volid = $2;
5990                 eval {
5991                     if ($volid =~ m|^/|) {
5992                         unlink $volid || die 'unlink failed\n';
5993                     } else {
5994                         PVE::Storage::vdisk_free($storecfg, $volid);
5995                     }
5996                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5997                 };
5998                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5999             } else {
6000                 print STDERR "unable to parse line in statfile - $line";
6001             }
6002         }
6003         $fd->close();
6004     }
6005 }
6006
6007 sub restore_archive {
6008     my ($archive, $vmid, $user, $opts) = @_;
6009
6010     my $format = $opts->{format};
6011     my $comp;
6012
6013     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6014         $format = 'tar' if !$format;
6015         $comp = 'gzip';
6016     } elsif ($archive =~ m/\.tar$/) {
6017         $format = 'tar' if !$format;
6018     } elsif ($archive =~ m/.tar.lzo$/) {
6019         $format = 'tar' if !$format;
6020         $comp = 'lzop';
6021     } elsif ($archive =~ m/\.vma$/) {
6022         $format = 'vma' if !$format;
6023     } elsif ($archive =~ m/\.vma\.gz$/) {
6024         $format = 'vma' if !$format;
6025         $comp = 'gzip';
6026     } elsif ($archive =~ m/\.vma\.lzo$/) {
6027         $format = 'vma' if !$format;
6028         $comp = 'lzop';
6029     } else {
6030         $format = 'vma' if !$format; # default
6031     }
6032
6033     # try to detect archive format
6034     if ($format eq 'tar') {
6035         return restore_tar_archive($archive, $vmid, $user, $opts);
6036     } else {
6037         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6038     }
6039 }
6040
6041 sub restore_update_config_line {
6042     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6043
6044     return if $line =~ m/^\#qmdump\#/;
6045     return if $line =~ m/^\#vzdump\#/;
6046     return if $line =~ m/^lock:/;
6047     return if $line =~ m/^unused\d+:/;
6048     return if $line =~ m/^parent:/;
6049
6050     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6051     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6052         # try to convert old 1.X settings
6053         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6054         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6055             my ($model, $macaddr) = split(/\=/, $devconfig);
6056             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6057             my $net = {
6058                 model => $model,
6059                 bridge => "vmbr$ind",
6060                 macaddr => $macaddr,
6061             };
6062             my $netstr = print_net($net);
6063
6064             print $outfd "net$cookie->{netcount}: $netstr\n";
6065             $cookie->{netcount}++;
6066         }
6067     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6068         my ($id, $netstr) = ($1, $2);
6069         my $net = parse_net($netstr);
6070         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6071         $netstr = print_net($net);
6072         print $outfd "$id: $netstr\n";
6073     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6074         my $virtdev = $1;
6075         my $value = $3;
6076         my $di = parse_drive($virtdev, $value);
6077         if (defined($di->{backup}) && !$di->{backup}) {
6078             print $outfd "#$line";
6079         } elsif ($map->{$virtdev}) {
6080             delete $di->{format}; # format can change on restore
6081             $di->{file} = $map->{$virtdev};
6082             $value = print_drive($vmid, $di);
6083             print $outfd "$virtdev: $value\n";
6084         } else {
6085             print $outfd $line;
6086         }
6087     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6088         my $vmgenid = $1;
6089         if ($vmgenid ne '0') {
6090             # always generate a new vmgenid if there was a valid one setup
6091             $vmgenid = generate_uuid();
6092         }
6093         print $outfd "vmgenid: $vmgenid\n";
6094     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6095         my ($uuid, $uuid_str);
6096         UUID::generate($uuid);
6097         UUID::unparse($uuid, $uuid_str);
6098         my $smbios1 = parse_smbios1($2);
6099         $smbios1->{uuid} = $uuid_str;
6100         print $outfd $1.print_smbios1($smbios1)."\n";
6101     } else {
6102         print $outfd $line;
6103     }
6104 }
6105
6106 sub scan_volids {
6107     my ($cfg, $vmid) = @_;
6108
6109     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6110
6111     my $volid_hash = {};
6112     foreach my $storeid (keys %$info) {
6113         foreach my $item (@{$info->{$storeid}}) {
6114             next if !($item->{volid} && $item->{size});
6115             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6116             $volid_hash->{$item->{volid}} = $item;
6117         }
6118     }
6119
6120     return $volid_hash;
6121 }
6122
6123 sub is_volume_in_use {
6124     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6125
6126     my $path = PVE::Storage::path($storecfg, $volid);
6127
6128     my $scan_config = sub {
6129         my ($cref, $snapname) = @_;
6130
6131         foreach my $key (keys %$cref) {
6132             my $value = $cref->{$key};
6133             if (is_valid_drivename($key)) {
6134                 next if $skip_drive && $key eq $skip_drive;
6135                 my $drive = parse_drive($key, $value);
6136                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6137                 return 1 if $volid eq $drive->{file};
6138                 if ($drive->{file} =~ m!^/!) {
6139                     return 1 if $drive->{file} eq $path;
6140                 } else {
6141                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6142                     next if !$storeid;
6143                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6144                     next if !$scfg;
6145                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6146                 }
6147             }
6148         }
6149
6150         return 0;
6151     };
6152
6153     return 1 if &$scan_config($conf);
6154
6155     undef $skip_drive;
6156
6157     foreach my $snapname (keys %{$conf->{snapshots}}) {
6158         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6159     }
6160
6161     return 0;
6162 }
6163
6164 sub update_disksize {
6165     my ($vmid, $conf, $volid_hash) = @_;
6166
6167     my $changes;
6168     my $prefix = "VM $vmid:";
6169
6170     # used and unused disks
6171     my $referenced = {};
6172
6173     # Note: it is allowed to define multiple storages with same path (alias), so
6174     # we need to check both 'volid' and real 'path' (two different volid can point
6175     # to the same path).
6176
6177     my $referencedpath = {};
6178
6179     # update size info
6180     foreach my $opt (keys %$conf) {
6181         if (is_valid_drivename($opt)) {
6182             my $drive = parse_drive($opt, $conf->{$opt});
6183             my $volid = $drive->{file};
6184             next if !$volid;
6185
6186             $referenced->{$volid} = 1;
6187             if ($volid_hash->{$volid} &&
6188                 (my $path = $volid_hash->{$volid}->{path})) {
6189                 $referencedpath->{$path} = 1;
6190             }
6191
6192             next if drive_is_cdrom($drive);
6193             next if !$volid_hash->{$volid};
6194
6195             $drive->{size} = $volid_hash->{$volid}->{size};
6196             my $new = print_drive($vmid, $drive);
6197             if ($new ne $conf->{$opt}) {
6198                 $changes = 1;
6199                 $conf->{$opt} = $new;
6200                 print "$prefix update disk '$opt' information.\n";
6201             }
6202         }
6203     }
6204
6205     # remove 'unusedX' entry if volume is used
6206     foreach my $opt (keys %$conf) {
6207         next if $opt !~ m/^unused\d+$/;
6208         my $volid = $conf->{$opt};
6209         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6210         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6211             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6212             $changes = 1;
6213             delete $conf->{$opt};
6214         }
6215
6216         $referenced->{$volid} = 1;
6217         $referencedpath->{$path} = 1 if $path;
6218     }
6219
6220     foreach my $volid (sort keys %$volid_hash) {
6221         next if $volid =~ m/vm-$vmid-state-/;
6222         next if $referenced->{$volid};
6223         my $path = $volid_hash->{$volid}->{path};
6224         next if !$path; # just to be sure
6225         next if $referencedpath->{$path};
6226         $changes = 1;
6227         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6228         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6229         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6230     }
6231
6232     return $changes;
6233 }
6234
6235 sub rescan {
6236     my ($vmid, $nolock, $dryrun) = @_;
6237
6238     my $cfg = PVE::Storage::config();
6239
6240     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6241     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6242     foreach my $stor (keys %{$cfg->{ids}}) {
6243         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6244     }
6245
6246     print "rescan volumes...\n";
6247     my $volid_hash = scan_volids($cfg, $vmid);
6248
6249     my $updatefn =  sub {
6250         my ($vmid) = @_;
6251
6252         my $conf = PVE::QemuConfig->load_config($vmid);
6253
6254         PVE::QemuConfig->check_lock($conf);
6255
6256         my $vm_volids = {};
6257         foreach my $volid (keys %$volid_hash) {
6258             my $info = $volid_hash->{$volid};
6259             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6260         }
6261
6262         my $changes = update_disksize($vmid, $conf, $vm_volids);
6263
6264         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6265     };
6266
6267     if (defined($vmid)) {
6268         if ($nolock) {
6269             &$updatefn($vmid);
6270         } else {
6271             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6272         }
6273     } else {
6274         my $vmlist = config_list();
6275         foreach my $vmid (keys %$vmlist) {
6276             if ($nolock) {
6277                 &$updatefn($vmid);
6278             } else {
6279                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6280             }
6281         }
6282     }
6283 }
6284
6285 sub restore_vma_archive {
6286     my ($archive, $vmid, $user, $opts, $comp) = @_;
6287
6288     my $readfrom = $archive;
6289
6290     my $cfg = PVE::Storage::config();
6291     my $commands = [];
6292     my $bwlimit = $opts->{bwlimit};
6293
6294     my $dbg_cmdstring = '';
6295     my $add_pipe = sub {
6296         my ($cmd) = @_;
6297         push @$commands, $cmd;
6298         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6299         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6300         $readfrom = '-';
6301     };
6302
6303     my $input = undef;
6304     if ($archive eq '-') {
6305         $input = '<&STDIN';
6306     } else {
6307         # If we use a backup from a PVE defined storage we also consider that
6308         # storage's rate limit:
6309         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6310         if (defined($volid)) {
6311             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6312             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6313             if ($readlimit) {
6314                 print STDERR "applying read rate limit: $readlimit\n";
6315                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6316                 $add_pipe->($cstream);
6317             }
6318         }
6319     }
6320
6321     if ($comp) {
6322         my $cmd;
6323         if ($comp eq 'gzip') {
6324             $cmd = ['zcat', $readfrom];
6325         } elsif ($comp eq 'lzop') {
6326             $cmd = ['lzop', '-d', '-c', $readfrom];
6327         } else {
6328             die "unknown compression method '$comp'\n";
6329         }
6330         $add_pipe->($cmd);
6331     }
6332
6333     my $tmpdir = "/var/tmp/vzdumptmp$$";
6334     rmtree $tmpdir;
6335
6336     # disable interrupts (always do cleanups)
6337     local $SIG{INT} =
6338         local $SIG{TERM} =
6339         local $SIG{QUIT} =
6340         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6341
6342     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6343     POSIX::mkfifo($mapfifo, 0600);
6344     my $fifofh;
6345
6346     my $openfifo = sub {
6347         open($fifofh, '>', $mapfifo) || die $!;
6348     };
6349
6350     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6351
6352     my $oldtimeout;
6353     my $timeout = 5;
6354
6355     my $devinfo = {};
6356
6357     my $rpcenv = PVE::RPCEnvironment::get();
6358
6359     my $conffile = PVE::QemuConfig->config_file($vmid);
6360     my $tmpfn = "$conffile.$$.tmp";
6361
6362     # Note: $oldconf is undef if VM does not exists
6363     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6364     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6365
6366     my %storage_limits;
6367
6368     my $print_devmap = sub {
6369         my $virtdev_hash = {};
6370
6371         my $cfgfn = "$tmpdir/qemu-server.conf";
6372
6373         # we can read the config - that is already extracted
6374         my $fh = IO::File->new($cfgfn, "r") ||
6375             "unable to read qemu-server.conf - $!\n";
6376
6377         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6378         if (-f $fwcfgfn) {
6379             my $pve_firewall_dir = '/etc/pve/firewall';
6380             mkdir $pve_firewall_dir; # make sure the dir exists
6381             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6382         }
6383
6384         while (defined(my $line = <$fh>)) {
6385             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6386                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6387                 die "archive does not contain data for drive '$virtdev'\n"
6388                     if !$devinfo->{$devname};
6389                 if (defined($opts->{storage})) {
6390                     $storeid = $opts->{storage} || 'local';
6391                 } elsif (!$storeid) {
6392                     $storeid = 'local';
6393                 }
6394                 $format = 'raw' if !$format;
6395                 $devinfo->{$devname}->{devname} = $devname;
6396                 $devinfo->{$devname}->{virtdev} = $virtdev;
6397                 $devinfo->{$devname}->{format} = $format;
6398                 $devinfo->{$devname}->{storeid} = $storeid;
6399
6400                 # check permission on storage
6401                 my $pool = $opts->{pool}; # todo: do we need that?
6402                 if ($user ne 'root@pam') {
6403                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6404                 }
6405
6406                 $storage_limits{$storeid} = $bwlimit;
6407
6408                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6409             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6410                 my $virtdev = $1;
6411                 my $drive = parse_drive($virtdev, $2);
6412                 if (drive_is_cloudinit($drive)) {
6413                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6414                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6415                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6416
6417                     my $d = {
6418                         format => $format,
6419                         storeid => $opts->{storage} // $storeid,
6420                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6421                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6422                         name => "vm-$vmid-cloudinit",
6423                         is_cloudinit => 1,
6424                     };
6425                     $virtdev_hash->{$virtdev} = $d;
6426                 }
6427             }
6428         }
6429
6430         foreach my $key (keys %storage_limits) {
6431             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6432             next if !$limit;
6433             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6434             $storage_limits{$key} = $limit * 1024;
6435         }
6436
6437         foreach my $devname (keys %$devinfo) {
6438             die "found no device mapping information for device '$devname'\n"
6439                 if !$devinfo->{$devname}->{virtdev};
6440         }
6441
6442         # create empty/temp config
6443         if ($oldconf) {
6444             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6445             foreach_drive($oldconf, sub {
6446                 my ($ds, $drive) = @_;
6447
6448                 return if drive_is_cdrom($drive, 1);
6449
6450                 my $volid = $drive->{file};
6451                 return if !$volid || $volid =~ m|^/|;
6452
6453                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6454                 return if !$path || !$owner || ($owner != $vmid);
6455
6456                 # Note: only delete disk we want to restore
6457                 # other volumes will become unused
6458                 if ($virtdev_hash->{$ds}) {
6459                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6460                     if (my $err = $@) {
6461                         warn $err;
6462                     }
6463                 }
6464             });
6465
6466             # delete vmstate files, after the restore we have no snapshots anymore
6467             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6468                 my $snap = $oldconf->{snapshots}->{$snapname};
6469                 if ($snap->{vmstate}) {
6470                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6471                     if (my $err = $@) {
6472                         warn $err;
6473                     }
6474                 }
6475             }
6476         }
6477
6478         my $map = {};
6479         foreach my $virtdev (sort keys %$virtdev_hash) {
6480             my $d = $virtdev_hash->{$virtdev};
6481             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6482             my $storeid = $d->{storeid};
6483             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6484
6485             my $map_opts = '';
6486             if (my $limit = $storage_limits{$storeid}) {
6487                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6488             }
6489
6490             # test if requested format is supported
6491             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6492             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6493             $d->{format} = $defFormat if !$supported;
6494
6495             my $name;
6496             if ($d->{is_cloudinit}) {
6497                 $name = $d->{name};
6498                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6499             }
6500
6501             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6502             print STDERR "new volume ID is '$volid'\n";
6503             $d->{volid} = $volid;
6504
6505             PVE::Storage::activate_volumes($cfg, [$volid]);
6506
6507             my $write_zeros = 1;
6508             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6509                 $write_zeros = 0;
6510             }
6511
6512             if (!$d->{is_cloudinit}) {
6513                 my $path = PVE::Storage::path($cfg, $volid);
6514
6515                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6516
6517                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6518             }
6519             $map->{$virtdev} = $volid;
6520         }
6521
6522         $fh->seek(0, 0) || die "seek failed - $!\n";
6523
6524         my $outfd = new IO::File ($tmpfn, "w") ||
6525             die "unable to write config for VM $vmid\n";
6526
6527         my $cookie = { netcount => 0 };
6528         while (defined(my $line = <$fh>)) {
6529             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6530         }
6531
6532         $fh->close();
6533         $outfd->close();
6534     };
6535
6536     eval {
6537         # enable interrupts
6538         local $SIG{INT} =
6539             local $SIG{TERM} =
6540             local $SIG{QUIT} =
6541             local $SIG{HUP} =
6542             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6543         local $SIG{ALRM} = sub { die "got timeout\n"; };
6544
6545         $oldtimeout = alarm($timeout);
6546
6547         my $parser = sub {
6548             my $line = shift;
6549
6550             print "$line\n";
6551
6552             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6553                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6554                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6555             } elsif ($line =~ m/^CTIME: /) {
6556                 # we correctly received the vma config, so we can disable
6557                 # the timeout now for disk allocation (set to 10 minutes, so
6558                 # that we always timeout if something goes wrong)
6559                 alarm(600);
6560                 &$print_devmap();
6561                 print $fifofh "done\n";
6562                 my $tmp = $oldtimeout || 0;
6563                 $oldtimeout = undef;
6564                 alarm($tmp);
6565                 close($fifofh);
6566             }
6567         };
6568
6569         print "restore vma archive: $dbg_cmdstring\n";
6570         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6571     };
6572     my $err = $@;
6573
6574     alarm($oldtimeout) if $oldtimeout;
6575
6576     my $vollist = [];
6577     foreach my $devname (keys %$devinfo) {
6578         my $volid = $devinfo->{$devname}->{volid};
6579         push @$vollist, $volid if $volid;
6580     }
6581
6582     PVE::Storage::deactivate_volumes($cfg, $vollist);
6583
6584     unlink $mapfifo;
6585
6586     if ($err) {
6587         rmtree $tmpdir;
6588         unlink $tmpfn;
6589
6590         foreach my $devname (keys %$devinfo) {
6591             my $volid = $devinfo->{$devname}->{volid};
6592             next if !$volid;
6593             eval {
6594                 if ($volid =~ m|^/|) {
6595                     unlink $volid || die 'unlink failed\n';
6596                 } else {
6597                     PVE::Storage::vdisk_free($cfg, $volid);
6598                 }
6599                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6600             };
6601             print STDERR "unable to cleanup '$volid' - $@" if $@;
6602         }
6603         die $err;
6604     }
6605
6606     rmtree $tmpdir;
6607
6608     rename($tmpfn, $conffile) ||
6609         die "unable to commit configuration file '$conffile'\n";
6610
6611     PVE::Cluster::cfs_update(); # make sure we read new file
6612
6613     eval { rescan($vmid, 1); };
6614     warn $@ if $@;
6615 }
6616
6617 sub restore_tar_archive {
6618     my ($archive, $vmid, $user, $opts) = @_;
6619
6620     if ($archive ne '-') {
6621         my $firstfile = tar_archive_read_firstfile($archive);
6622         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6623             if $firstfile ne 'qemu-server.conf';
6624     }
6625
6626     my $storecfg = PVE::Storage::config();
6627
6628     # destroy existing data - keep empty config
6629     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6630     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6631
6632     my $tocmd = "/usr/lib/qemu-server/qmextract";
6633
6634     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6635     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6636     $tocmd .= ' --prealloc' if $opts->{prealloc};
6637     $tocmd .= ' --info' if $opts->{info};
6638
6639     # tar option "xf" does not autodetect compression when read from STDIN,
6640     # so we pipe to zcat
6641     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6642         PVE::Tools::shellquote("--to-command=$tocmd");
6643
6644     my $tmpdir = "/var/tmp/vzdumptmp$$";
6645     mkpath $tmpdir;
6646
6647     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6648     local $ENV{VZDUMP_VMID} = $vmid;
6649     local $ENV{VZDUMP_USER} = $user;
6650
6651     my $conffile = PVE::QemuConfig->config_file($vmid);
6652     my $tmpfn = "$conffile.$$.tmp";
6653
6654     # disable interrupts (always do cleanups)
6655     local $SIG{INT} =
6656         local $SIG{TERM} =
6657         local $SIG{QUIT} =
6658         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6659
6660     eval {
6661         # enable interrupts
6662         local $SIG{INT} =
6663             local $SIG{TERM} =
6664             local $SIG{QUIT} =
6665             local $SIG{HUP} =
6666             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6667
6668         if ($archive eq '-') {
6669             print "extracting archive from STDIN\n";
6670             run_command($cmd, input => "<&STDIN");
6671         } else {
6672             print "extracting archive '$archive'\n";
6673             run_command($cmd);
6674         }
6675
6676         return if $opts->{info};
6677
6678         # read new mapping
6679         my $map = {};
6680         my $statfile = "$tmpdir/qmrestore.stat";
6681         if (my $fd = IO::File->new($statfile, "r")) {
6682             while (defined (my $line = <$fd>)) {
6683                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6684                     $map->{$1} = $2 if $1;
6685                 } else {
6686                     print STDERR "unable to parse line in statfile - $line\n";
6687                 }
6688             }
6689             $fd->close();
6690         }
6691
6692         my $confsrc = "$tmpdir/qemu-server.conf";
6693
6694         my $srcfd = new IO::File($confsrc, "r") ||
6695             die "unable to open file '$confsrc'\n";
6696
6697         my $outfd = new IO::File ($tmpfn, "w") ||
6698             die "unable to write config for VM $vmid\n";
6699
6700         my $cookie = { netcount => 0 };
6701         while (defined (my $line = <$srcfd>)) {
6702             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6703         }
6704
6705         $srcfd->close();
6706         $outfd->close();
6707     };
6708     my $err = $@;
6709
6710     if ($err) {
6711
6712         unlink $tmpfn;
6713
6714         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6715
6716         die $err;
6717     }
6718
6719     rmtree $tmpdir;
6720
6721     rename $tmpfn, $conffile ||
6722         die "unable to commit configuration file '$conffile'\n";
6723
6724     PVE::Cluster::cfs_update(); # make sure we read new file
6725
6726     eval { rescan($vmid, 1); };
6727     warn $@ if $@;
6728 };
6729
6730 sub foreach_storage_used_by_vm {
6731     my ($conf, $func) = @_;
6732
6733     my $sidhash = {};
6734
6735     foreach_drive($conf, sub {
6736         my ($ds, $drive) = @_;
6737         return if drive_is_cdrom($drive);
6738
6739         my $volid = $drive->{file};
6740
6741         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6742         $sidhash->{$sid} = $sid if $sid;
6743     });
6744
6745     foreach my $sid (sort keys %$sidhash) {
6746         &$func($sid);
6747     }
6748 }
6749
6750 sub do_snapshots_with_qemu {
6751     my ($storecfg, $volid) = @_;
6752
6753     my $storage_name = PVE::Storage::parse_volume_id($volid);
6754     my $scfg = $storecfg->{ids}->{$storage_name};
6755
6756     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6757         return 1;
6758     }
6759
6760     if ($volid =~ m/\.(qcow2|qed)$/){
6761         return 1;
6762     }
6763
6764     return undef;
6765 }
6766
6767 sub qga_check_running {
6768     my ($vmid, $nowarn) = @_;
6769
6770     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6771     if ($@) {
6772         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6773         return 0;
6774     }
6775     return 1;
6776 }
6777
6778 sub template_create {
6779     my ($vmid, $conf, $disk) = @_;
6780
6781     my $storecfg = PVE::Storage::config();
6782
6783     foreach_drive($conf, sub {
6784         my ($ds, $drive) = @_;
6785
6786         return if drive_is_cdrom($drive);
6787         return if $disk && $ds ne $disk;
6788
6789         my $volid = $drive->{file};
6790         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6791
6792         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6793         $drive->{file} = $voliddst;
6794         $conf->{$ds} = print_drive($vmid, $drive);
6795         PVE::QemuConfig->write_config($vmid, $conf);
6796     });
6797 }
6798
6799 sub convert_iscsi_path {
6800     my ($path) = @_;
6801
6802     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6803         my $portal = $1;
6804         my $target = $2;
6805         my $lun = $3;
6806
6807         my $initiator_name = get_initiator_name();
6808
6809         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6810                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6811     }
6812
6813     die "cannot convert iscsi path '$path', unkown format\n";
6814 }
6815
6816 sub qemu_img_convert {
6817     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6818
6819     my $storecfg = PVE::Storage::config();
6820     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6821     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6822
6823     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6824
6825     my $cachemode;
6826     my $src_path;
6827     my $src_is_iscsi = 0;
6828     my $src_format = 'raw';
6829
6830     if ($src_storeid) {
6831         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6832         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6833         $src_format = qemu_img_format($src_scfg, $src_volname);
6834         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6835         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6836         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6837     } elsif (-f $src_volid) {
6838         $src_path = $src_volid;
6839         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6840             $src_format = $1;
6841         }
6842     }
6843
6844     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6845
6846     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6847     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6848     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6849     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6850
6851     my $cmd = [];
6852     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6853     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6854     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6855     push @$cmd, '-T', $cachemode if defined($cachemode);
6856
6857     if ($src_is_iscsi) {
6858         push @$cmd, '--image-opts';
6859         $src_path = convert_iscsi_path($src_path);
6860     } else {
6861         push @$cmd, '-f', $src_format;
6862     }
6863
6864     if ($dst_is_iscsi) {
6865         push @$cmd, '--target-image-opts';
6866         $dst_path = convert_iscsi_path($dst_path);
6867     } else {
6868         push @$cmd, '-O', $dst_format;
6869     }
6870
6871     push @$cmd, $src_path;
6872
6873     if (!$dst_is_iscsi && $is_zero_initialized) {
6874         push @$cmd, "zeroinit:$dst_path";
6875     } else {
6876         push @$cmd, $dst_path;
6877     }
6878
6879     my $parser = sub {
6880         my $line = shift;
6881         if($line =~ m/\((\S+)\/100\%\)/){
6882             my $percent = $1;
6883             my $transferred = int($size * $percent / 100);
6884             my $remaining = $size - $transferred;
6885
6886             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6887         }
6888
6889     };
6890
6891     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6892     my $err = $@;
6893     die "copy failed: $err" if $err;
6894 }
6895
6896 sub qemu_img_format {
6897     my ($scfg, $volname) = @_;
6898
6899     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6900         return $1;
6901     } else {
6902         return "raw";
6903     }
6904 }
6905
6906 sub qemu_drive_mirror {
6907     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6908
6909     $jobs = {} if !$jobs;
6910
6911     my $qemu_target;
6912     my $format;
6913     $jobs->{"drive-$drive"} = {};
6914
6915     if ($dst_volid =~ /^nbd:/) {
6916         $qemu_target = $dst_volid;
6917         $format = "nbd";
6918     } else {
6919         my $storecfg = PVE::Storage::config();
6920         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6921
6922         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6923
6924         $format = qemu_img_format($dst_scfg, $dst_volname);
6925
6926         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6927
6928         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6929     }
6930
6931     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6932     $opts->{format} = $format if $format;
6933
6934     if (defined($bwlimit)) {
6935         $opts->{speed} = $bwlimit * 1024;
6936         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6937     } else {
6938         print "drive mirror is starting for drive-$drive\n";
6939     }
6940
6941     # if a job already runs for this device we get an error, catch it for cleanup
6942     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6943     if (my $err = $@) {
6944         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6945         warn "$@\n" if $@;
6946         die "mirroring error: $err\n";
6947     }
6948
6949     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6950 }
6951
6952 sub qemu_drive_mirror_monitor {
6953     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6954
6955     eval {
6956         my $err_complete = 0;
6957
6958         while (1) {
6959             die "storage migration timed out\n" if $err_complete > 300;
6960
6961             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6962
6963             my $running_mirror_jobs = {};
6964             foreach my $stat (@$stats) {
6965                 next if $stat->{type} ne 'mirror';
6966                 $running_mirror_jobs->{$stat->{device}} = $stat;
6967             }
6968
6969             my $readycounter = 0;
6970
6971             foreach my $job (keys %$jobs) {
6972
6973                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6974                     print "$job : finished\n";
6975                     delete $jobs->{$job};
6976                     next;
6977                 }
6978
6979                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6980
6981                 my $busy = $running_mirror_jobs->{$job}->{busy};
6982                 my $ready = $running_mirror_jobs->{$job}->{ready};
6983                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6984                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6985                     my $remaining = $total - $transferred;
6986                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6987
6988                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6989                 }
6990
6991                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6992             }
6993
6994             last if scalar(keys %$jobs) == 0;
6995
6996             if ($readycounter == scalar(keys %$jobs)) {
6997                 print "all mirroring jobs are ready \n";
6998                 last if $skipcomplete; #do the complete later
6999
7000                 if ($vmiddst && $vmiddst != $vmid) {
7001                     my $agent_running = $qga && qga_check_running($vmid);
7002                     if ($agent_running) {
7003                         print "freeze filesystem\n";
7004                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7005                     } else {
7006                         print "suspend vm\n";
7007                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7008                     }
7009
7010                     # if we clone a disk for a new target vm, we don't switch the disk
7011                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7012
7013                     if ($agent_running) {
7014                         print "unfreeze filesystem\n";
7015                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7016                     } else {
7017                         print "resume vm\n";
7018                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7019                     }
7020
7021                     last;
7022                 } else {
7023
7024                     foreach my $job (keys %$jobs) {
7025                         # try to switch the disk if source and destination are on the same guest
7026                         print "$job: Completing block job...\n";
7027
7028                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7029                         if ($@ =~ m/cannot be completed/) {
7030                             print "$job: Block job cannot be completed, try again.\n";
7031                             $err_complete++;
7032                         }else {
7033                             print "$job: Completed successfully.\n";
7034                             $jobs->{$job}->{complete} = 1;
7035                         }
7036                     }
7037                 }
7038             }
7039             sleep 1;
7040         }
7041     };
7042     my $err = $@;
7043
7044     if ($err) {
7045         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7046         die "mirroring error: $err";
7047     }
7048
7049 }
7050
7051 sub qemu_blockjobs_cancel {
7052     my ($vmid, $jobs) = @_;
7053
7054     foreach my $job (keys %$jobs) {
7055         print "$job: Cancelling block job\n";
7056         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7057         $jobs->{$job}->{cancel} = 1;
7058     }
7059
7060     while (1) {
7061         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7062
7063         my $running_jobs = {};
7064         foreach my $stat (@$stats) {
7065             $running_jobs->{$stat->{device}} = $stat;
7066         }
7067
7068         foreach my $job (keys %$jobs) {
7069
7070             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7071                 print "$job: Done.\n";
7072                 delete $jobs->{$job};
7073             }
7074         }
7075
7076         last if scalar(keys %$jobs) == 0;
7077
7078         sleep 1;
7079     }
7080 }
7081
7082 sub clone_disk {
7083     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7084         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7085
7086     my $newvolid;
7087
7088     if (!$full) {
7089         print "create linked clone of drive $drivename ($drive->{file})\n";
7090         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7091         push @$newvollist, $newvolid;
7092     } else {
7093
7094         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7095         $storeid = $storage if $storage;
7096
7097         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7098         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7099
7100         print "create full clone of drive $drivename ($drive->{file})\n";
7101         my $name = undef;
7102         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7103         push @$newvollist, $newvolid;
7104
7105         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7106
7107         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7108         if (!$running || $snapname) {
7109             # TODO: handle bwlimits
7110             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7111         } else {
7112
7113             my $kvmver = get_running_qemu_version ($vmid);
7114             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7115                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7116                     if $drive->{iothread};
7117             }
7118
7119             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7120         }
7121     }
7122
7123     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7124
7125     my $disk = $drive;
7126     $disk->{format} = undef;
7127     $disk->{file} = $newvolid;
7128     $disk->{size} = $size;
7129
7130     return $disk;
7131 }
7132
7133 # this only works if VM is running
7134 sub get_current_qemu_machine {
7135     my ($vmid) = @_;
7136
7137     my $cmd = { execute => 'query-machines', arguments => {} };
7138     my $res = vm_qmp_command($vmid, $cmd);
7139
7140     my ($current, $default);
7141     foreach my $e (@$res) {
7142         $default = $e->{name} if $e->{'is-default'};
7143         $current = $e->{name} if $e->{'is-current'};
7144     }
7145
7146     # fallback to the default machine if current is not supported by qemu
7147     return $current || $default || 'pc';
7148 }
7149
7150 sub get_running_qemu_version {
7151     my ($vmid) = @_;
7152     my $cmd = { execute => 'query-version', arguments => {} };
7153     my $res = vm_qmp_command($vmid, $cmd);
7154     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7155 }
7156
7157 sub qemu_machine_feature_enabled {
7158     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7159
7160     my $current_major;
7161     my $current_minor;
7162
7163     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7164
7165         $current_major = $3;
7166         $current_minor = $4;
7167
7168     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7169
7170         $current_major = $1;
7171         $current_minor = $2;
7172     }
7173
7174     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7175 }
7176
7177 # gets in pairs the versions you want to compares, i.e.:
7178 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7179 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7180 sub version_cmp {
7181     my @versions = @_;
7182
7183     my $size = scalar(@versions);
7184
7185     return 0 if $size == 0;
7186     die "cannot compare odd count of versions" if $size & 1;
7187
7188     for (my $i = 0; $i < $size; $i += 2) {
7189         my ($a, $b) = splice(@versions, 0, 2);
7190         $a //= 0;
7191         $b //= 0;
7192
7193         return 1 if $a > $b;
7194         return -1 if $a < $b;
7195     }
7196     return 0;
7197 }
7198
7199 # dies if a) VM not running or not exisiting b) Version query failed
7200 # So, any defined return value is valid, any invalid state can be caught by eval
7201 sub runs_at_least_qemu_version {
7202     my ($vmid, $major, $minor, $extra) = @_;
7203
7204     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7205     die "could not query currently running version for VM $vmid\n" if !defined($v);
7206     $v = $v->{qemu};
7207
7208     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7209 }
7210
7211 sub qemu_machine_pxe {
7212     my ($vmid, $conf) = @_;
7213
7214     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7215
7216     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7217         $machine .= '.pxe';
7218     }
7219
7220     return $machine;
7221 }
7222
7223 sub qemu_use_old_bios_files {
7224     my ($machine_type) = @_;
7225
7226     return if !$machine_type;
7227
7228     my $use_old_bios_files = undef;
7229
7230     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7231         $machine_type = $1;
7232         $use_old_bios_files = 1;
7233     } else {
7234         my $kvmver = kvm_user_version();
7235         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7236         # load new efi bios files on migration. So this hack is required to allow
7237         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7238         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7239         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7240     }
7241
7242     return ($use_old_bios_files, $machine_type);
7243 }
7244
7245 sub create_efidisk($$$$$) {
7246     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7247
7248     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7249     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7250
7251     my $vars_size_b = -s $ovmf_vars;
7252     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7253     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7254     PVE::Storage::activate_volumes($storecfg, [$volid]);
7255
7256     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7257
7258     return ($volid, $vars_size);
7259 }
7260
7261 sub vm_iothreads_list {
7262     my ($vmid) = @_;
7263
7264     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7265
7266     my $iothreads = {};
7267     foreach my $iothread (@$res) {
7268         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7269     }
7270
7271     return $iothreads;
7272 }
7273
7274 sub scsihw_infos {
7275     my ($conf, $drive) = @_;
7276
7277     my $maxdev = 0;
7278
7279     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7280         $maxdev = 7;
7281     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7282         $maxdev = 1;
7283     } else {
7284         $maxdev = 256;
7285     }
7286
7287     my $controller = int($drive->{index} / $maxdev);
7288     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7289
7290     return ($maxdev, $controller, $controller_prefix);
7291 }
7292
7293 sub add_hyperv_enlightenments {
7294     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7295
7296     return if $winversion < 6;
7297     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7298
7299     if ($gpu_passthrough || defined($hv_vendor_id)) {
7300         $hv_vendor_id //= 'proxmox';
7301         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7302     }
7303
7304     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7305         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7306         push @$cpuFlags , 'hv_vapic';
7307         push @$cpuFlags , 'hv_time';
7308     } else {
7309         push @$cpuFlags , 'hv_spinlocks=0xffff';
7310     }
7311
7312     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7313         push @$cpuFlags , 'hv_reset';
7314         push @$cpuFlags , 'hv_vpindex';
7315         push @$cpuFlags , 'hv_runtime';
7316     }
7317
7318     if ($winversion >= 7) {
7319         push @$cpuFlags , 'hv_relaxed';
7320
7321         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7322             push @$cpuFlags , 'hv_synic';
7323             push @$cpuFlags , 'hv_stimer';
7324         }
7325
7326         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7327             push @$cpuFlags , 'hv_ipi';
7328         }
7329     }
7330 }
7331
7332 sub windows_version {
7333     my ($ostype) = @_;
7334
7335     return 0 if !$ostype;
7336
7337     my $winversion = 0;
7338
7339     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7340         $winversion = 5;
7341     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7342         $winversion = 6;
7343     } elsif ($ostype =~ m/^win(\d+)$/) {
7344         $winversion = $1;
7345     }
7346
7347     return $winversion;
7348 }
7349
7350 sub resolve_dst_disk_format {
7351         my ($storecfg, $storeid, $src_volname, $format) = @_;
7352         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7353
7354         if (!$format) {
7355             # if no target format is specified, use the source disk format as hint
7356             if ($src_volname) {
7357                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7358                 $format = qemu_img_format($scfg, $src_volname);
7359             } else {
7360                 return $defFormat;
7361             }
7362         }
7363
7364         # test if requested format is supported - else use default
7365         my $supported = grep { $_ eq $format } @$validFormats;
7366         $format = $defFormat if !$supported;
7367         return $format;
7368 }
7369
7370 sub resolve_first_disk {
7371     my $conf = shift;
7372     my @disks = PVE::QemuServer::valid_drive_names();
7373     my $firstdisk;
7374     foreach my $ds (reverse @disks) {
7375         next if !$conf->{$ds};
7376         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7377         next if PVE::QemuServer::drive_is_cdrom($disk);
7378         $firstdisk = $ds;
7379     }
7380     return $firstdisk;
7381 }
7382
7383 sub generate_uuid {
7384     my ($uuid, $uuid_str);
7385     UUID::generate($uuid);
7386     UUID::unparse($uuid, $uuid_str);
7387     return $uuid_str;
7388 }
7389
7390 sub generate_smbios1_uuid {
7391     return "uuid=".generate_uuid();
7392 }
7393
7394 sub nbd_stop {
7395     my ($vmid) = @_;
7396
7397     vm_mon_cmd($vmid, 'nbd-server-stop');
7398 }
7399
7400 sub create_reboot_request {
7401     my ($vmid) = @_;
7402     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7403         or die "failed to create reboot trigger file: $!\n";
7404     close($fh);
7405 }
7406
7407 sub clear_reboot_request {
7408     my ($vmid) = @_;
7409     my $path = "/run/qemu-server/$vmid.reboot";
7410     my $res = 0;
7411
7412     $res = unlink($path);
7413     die "could not remove reboot request for $vmid: $!"
7414         if !$res && $! != POSIX::ENOENT;
7415
7416     return $res;
7417 }
7418
7419 # bash completion helper
7420
7421 sub complete_backup_archives {
7422     my ($cmdname, $pname, $cvalue) = @_;
7423
7424     my $cfg = PVE::Storage::config();
7425
7426     my $storeid;
7427
7428     if ($cvalue =~ m/^([^:]+):/) {
7429         $storeid = $1;
7430     }
7431
7432     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7433
7434     my $res = [];
7435     foreach my $id (keys %$data) {
7436         foreach my $item (@{$data->{$id}}) {
7437             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7438             push @$res, $item->{volid} if defined($item->{volid});
7439         }
7440     }
7441
7442     return $res;
7443 }
7444
7445 my $complete_vmid_full = sub {
7446     my ($running) = @_;
7447
7448     my $idlist = vmstatus();
7449
7450     my $res = [];
7451
7452     foreach my $id (keys %$idlist) {
7453         my $d = $idlist->{$id};
7454         if (defined($running)) {
7455             next if $d->{template};
7456             next if $running && $d->{status} ne 'running';
7457             next if !$running && $d->{status} eq 'running';
7458         }
7459         push @$res, $id;
7460
7461     }
7462     return $res;
7463 };
7464
7465 sub complete_vmid {
7466     return &$complete_vmid_full();
7467 }
7468
7469 sub complete_vmid_stopped {
7470     return &$complete_vmid_full(0);
7471 }
7472
7473 sub complete_vmid_running {
7474     return &$complete_vmid_full(1);
7475 }
7476
7477 sub complete_storage {
7478
7479     my $cfg = PVE::Storage::config();
7480     my $ids = $cfg->{ids};
7481
7482     my $res = [];
7483     foreach my $sid (keys %$ids) {
7484         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7485         next if !$ids->{$sid}->{content}->{images};
7486         push @$res, $sid;
7487     }
7488
7489     return $res;
7490 }
7491
7492 1;