PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1902             if ($path =~ m/^iscsi\:\/\// &&
1903                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1904                 $devicetype = 'generic';
1905             }
1906         }
1907
1908         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         } else {
1911             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1912         }
1913
1914         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1915             $device .= ",rotation_rate=1";
1916         }
1917         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1918
1919     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1920         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1921         my $controller = int($drive->{index} / $maxdev);
1922         my $unit = $drive->{index} % $maxdev;
1923         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1924
1925         $device = "ide-$devicetype";
1926         if ($drive->{interface} eq 'ide') {
1927             $device .= ",bus=ide.$controller,unit=$unit";
1928         } else {
1929             $device .= ",bus=ahci$controller.$unit";
1930         }
1931         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1932
1933         if ($devicetype eq 'hd') {
1934             if (my $model = $drive->{model}) {
1935                 $model = URI::Escape::uri_unescape($model);
1936                 $device .= ",model=$model";
1937             }
1938             if ($drive->{ssd}) {
1939                 $device .= ",rotation_rate=1";
1940             }
1941         }
1942         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1943     } elsif ($drive->{interface} eq 'usb') {
1944         die "implement me";
1945         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1946     } else {
1947         die "unsupported interface type";
1948     }
1949
1950     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1951
1952     if (my $serial = $drive->{serial}) {
1953         $serial = URI::Escape::uri_unescape($serial);
1954         $device .= ",serial=$serial";
1955     }
1956
1957
1958     return $device;
1959 }
1960
1961 sub get_initiator_name {
1962     my $initiator;
1963
1964     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1965     while (defined(my $line = <$fh>)) {
1966         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1967         $initiator = $1;
1968         last;
1969     }
1970     $fh->close();
1971
1972     return $initiator;
1973 }
1974
1975 sub print_drive_full {
1976     my ($storecfg, $vmid, $drive) = @_;
1977
1978     my $path;
1979     my $volid = $drive->{file};
1980     my $format;
1981
1982     if (drive_is_cdrom($drive)) {
1983         $path = get_iso_path($storecfg, $vmid, $volid);
1984     } else {
1985         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1986         if ($storeid) {
1987             $path = PVE::Storage::path($storecfg, $volid);
1988             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1989             $format = qemu_img_format($scfg, $volname);
1990         } else {
1991             $path = $volid;
1992             $format = "raw";
1993         }
1994    }
1995
1996     my $opts = '';
1997     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1998     foreach my $o (@qemu_drive_options) {
1999         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2000     }
2001
2002     # snapshot only accepts on|off
2003     if (defined($drive->{snapshot})) {
2004         my $v = $drive->{snapshot} ? 'on' : 'off';
2005         $opts .= ",snapshot=$v";
2006     }
2007
2008     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2009         my ($dir, $qmpname) = @$type;
2010         if (my $v = $drive->{"mbps$dir"}) {
2011             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2012         }
2013         if (my $v = $drive->{"mbps${dir}_max"}) {
2014             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2015         }
2016         if (my $v = $drive->{"bps${dir}_max_length"}) {
2017             $opts .= ",throttling.bps$qmpname-max-length=$v";
2018         }
2019         if (my $v = $drive->{"iops${dir}"}) {
2020             $opts .= ",throttling.iops$qmpname=$v";
2021         }
2022         if (my $v = $drive->{"iops${dir}_max"}) {
2023             $opts .= ",throttling.iops$qmpname-max=$v";
2024         }
2025         if (my $v = $drive->{"iops${dir}_max_length"}) {
2026             $opts .= ",throttling.iops$qmpname-max-length=$v";
2027         }
2028     }
2029
2030     $opts .= ",format=$format" if $format && !$drive->{format};
2031
2032     my $cache_direct = 0;
2033
2034     if (my $cache = $drive->{cache}) {
2035         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2036     } elsif (!drive_is_cdrom($drive)) {
2037         $opts .= ",cache=none";
2038         $cache_direct = 1;
2039     }
2040
2041     # aio native works only with O_DIRECT
2042     if (!$drive->{aio}) {
2043         if($cache_direct) {
2044             $opts .= ",aio=native";
2045         } else {
2046             $opts .= ",aio=threads";
2047         }
2048     }
2049
2050     if (!drive_is_cdrom($drive)) {
2051         my $detectzeroes;
2052         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2053             $detectzeroes = 'off';
2054         } elsif ($drive->{discard}) {
2055             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2056         } else {
2057             # This used to be our default with discard not being specified:
2058             $detectzeroes = 'on';
2059         }
2060         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2061     }
2062
2063     my $pathinfo = $path ? "file=$path," : '';
2064
2065     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2066 }
2067
2068 sub print_netdevice_full {
2069     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2070
2071     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2072
2073     my $device = $net->{model};
2074     if ($net->{model} eq 'virtio') {
2075          $device = 'virtio-net-pci';
2076      };
2077
2078     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2079     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2080     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2081         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2082         my $vectors = $net->{queues} * 2 + 2;
2083         $tmpstr .= ",vectors=$vectors,mq=on";
2084     }
2085     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2086
2087     if ($use_old_bios_files) {
2088         my $romfile;
2089         if ($device eq 'virtio-net-pci') {
2090             $romfile = 'pxe-virtio.rom';
2091         } elsif ($device eq 'e1000') {
2092             $romfile = 'pxe-e1000.rom';
2093         } elsif ($device eq 'ne2k') {
2094             $romfile = 'pxe-ne2k_pci.rom';
2095         } elsif ($device eq 'pcnet') {
2096             $romfile = 'pxe-pcnet.rom';
2097         } elsif ($device eq 'rtl8139') {
2098             $romfile = 'pxe-rtl8139.rom';
2099         }
2100         $tmpstr .= ",romfile=$romfile" if $romfile;
2101     }
2102
2103     return $tmpstr;
2104 }
2105
2106 sub print_netdev_full {
2107     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2108
2109     my $i = '';
2110     if ($netid =~ m/^net(\d+)$/) {
2111         $i = int($1);
2112     }
2113
2114     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2115
2116     my $ifname = "tap${vmid}i$i";
2117
2118     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2119     die "interface name '$ifname' is too long (max 15 character)\n"
2120         if length($ifname) >= 16;
2121
2122     my $vhostparam = '';
2123     if (is_native($arch)) {
2124         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2125     }
2126
2127     my $vmname = $conf->{name} || "vm$vmid";
2128
2129     my $netdev = "";
2130     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2131
2132     if ($net->{bridge}) {
2133         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2134     } else {
2135         $netdev = "type=user,id=$netid,hostname=$vmname";
2136     }
2137
2138     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2139
2140     return $netdev;
2141 }
2142
2143
2144 sub print_cpu_device {
2145     my ($conf, $id) = @_;
2146
2147     my $kvm = $conf->{kvm} // 1;
2148     my $cpu = $kvm ? "kvm64" : "qemu64";
2149     if (my $cputype = $conf->{cpu}) {
2150         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2151             or die "Cannot parse cpu description: $cputype\n";
2152         $cpu = $cpuconf->{cputype};
2153     }
2154
2155     my $cores = $conf->{cores} || 1;
2156
2157     my $current_core = ($id - 1) % $cores;
2158     my $current_socket = int(($id - 1 - $current_core)/$cores);
2159
2160     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2161 }
2162
2163 my $vga_map = {
2164     'cirrus' => 'cirrus-vga',
2165     'std' => 'VGA',
2166     'vmware' => 'vmware-svga',
2167     'virtio' => 'virtio-vga',
2168 };
2169
2170 sub print_vga_device {
2171     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2172
2173     my $type = $vga_map->{$vga->{type}};
2174     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2175         $type = 'virtio-gpu';
2176     }
2177     my $vgamem_mb = $vga->{memory};
2178     if ($qxlnum) {
2179         $type = $id ? 'qxl' : 'qxl-vga';
2180     }
2181     die "no devicetype for $vga->{type}\n" if !$type;
2182
2183     my $memory = "";
2184     if ($vgamem_mb) {
2185         if ($vga->{type} eq 'virtio') {
2186             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2187             $memory = ",max_hostmem=$bytes";
2188         } elsif ($qxlnum) {
2189             # from https://www.spice-space.org/multiple-monitors.html
2190             $memory = ",vgamem_mb=$vga->{memory}";
2191             my $ram = $vgamem_mb * 4;
2192             my $vram = $vgamem_mb * 2;
2193             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2194         } else {
2195             $memory = ",vgamem_mb=$vga->{memory}";
2196         }
2197     } elsif ($qxlnum && $id) {
2198         $memory = ",ram_size=67108864,vram_size=33554432";
2199     }
2200
2201     my $q35 = machine_type_is_q35($conf);
2202     my $vgaid = "vga" . ($id // '');
2203     my $pciaddr;
2204
2205     if ($q35 && $vgaid eq 'vga') {
2206         # the first display uses pcie.0 bus on q35 machines
2207         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2208     } else {
2209         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2210     }
2211
2212     return "$type,id=${vgaid}${memory}${pciaddr}";
2213 }
2214
2215 sub drive_is_cloudinit {
2216     my ($drive) = @_;
2217     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2218 }
2219
2220 sub drive_is_cdrom {
2221     my ($drive, $exclude_cloudinit) = @_;
2222
2223     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2224
2225     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2226
2227 }
2228
2229 sub parse_number_sets {
2230     my ($set) = @_;
2231     my $res = [];
2232     foreach my $part (split(/;/, $set)) {
2233         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2234             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2235             push @$res, [ $1, $2 ];
2236         } else {
2237             die "invalid range: $part\n";
2238         }
2239     }
2240     return $res;
2241 }
2242
2243 sub parse_numa {
2244     my ($data) = @_;
2245
2246     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2247     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2248     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2249     return $res;
2250 }
2251
2252 sub parse_hostpci {
2253     my ($value) = @_;
2254
2255     return undef if !$value;
2256
2257     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2258
2259     my @idlist = split(/;/, $res->{host});
2260     delete $res->{host};
2261     foreach my $id (@idlist) {
2262         if ($id =~ m/\./) { # full id 00:00.1
2263             push @{$res->{pciid}}, {
2264                 id => $id,
2265             };
2266         } else { # partial id 00:00
2267             $res->{pciid} = PVE::SysFSTools::lspci($id);
2268         }
2269     }
2270     return $res;
2271 }
2272
2273 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2274 sub parse_net {
2275     my ($data) = @_;
2276
2277     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2278     if ($@) {
2279         warn $@;
2280         return undef;
2281     }
2282     if (!defined($res->{macaddr})) {
2283         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2284         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2285     }
2286     return $res;
2287 }
2288
2289 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2290 sub parse_ipconfig {
2291     my ($data) = @_;
2292
2293     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2294     if ($@) {
2295         warn $@;
2296         return undef;
2297     }
2298
2299     if ($res->{gw} && !$res->{ip}) {
2300         warn 'gateway specified without specifying an IP address';
2301         return undef;
2302     }
2303     if ($res->{gw6} && !$res->{ip6}) {
2304         warn 'IPv6 gateway specified without specifying an IPv6 address';
2305         return undef;
2306     }
2307     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2308         warn 'gateway specified together with DHCP';
2309         return undef;
2310     }
2311     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2312         # gw6 + auto/dhcp
2313         warn "IPv6 gateway specified together with $res->{ip6} address";
2314         return undef;
2315     }
2316
2317     if (!$res->{ip} && !$res->{ip6}) {
2318         return { ip => 'dhcp', ip6 => 'dhcp' };
2319     }
2320
2321     return $res;
2322 }
2323
2324 sub print_net {
2325     my $net = shift;
2326
2327     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2328 }
2329
2330 sub add_random_macs {
2331     my ($settings) = @_;
2332
2333     foreach my $opt (keys %$settings) {
2334         next if $opt !~ m/^net(\d+)$/;
2335         my $net = parse_net($settings->{$opt});
2336         next if !$net;
2337         $settings->{$opt} = print_net($net);
2338     }
2339 }
2340
2341 sub vm_is_volid_owner {
2342     my ($storecfg, $vmid, $volid) = @_;
2343
2344     if ($volid !~  m|^/|) {
2345         my ($path, $owner);
2346         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2347         if ($owner && ($owner == $vmid)) {
2348             return 1;
2349         }
2350     }
2351
2352     return undef;
2353 }
2354
2355 sub vmconfig_register_unused_drive {
2356     my ($storecfg, $vmid, $conf, $drive) = @_;
2357
2358     if (drive_is_cloudinit($drive)) {
2359         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2360         warn $@ if $@;
2361     } elsif (!drive_is_cdrom($drive)) {
2362         my $volid = $drive->{file};
2363         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2364             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2365         }
2366     }
2367 }
2368
2369 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2370 my $smbios1_fmt = {
2371     uuid => {
2372         type => 'string',
2373         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2374         format_description => 'UUID',
2375         description => "Set SMBIOS1 UUID.",
2376         optional => 1,
2377     },
2378     version => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 version.",
2383         optional => 1,
2384     },
2385     serial => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 serial number.",
2390         optional => 1,
2391     },
2392     manufacturer => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 manufacturer.",
2397         optional => 1,
2398     },
2399     product => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 product ID.",
2404         optional => 1,
2405     },
2406     sku => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 SKU string.",
2411         optional => 1,
2412     },
2413     family => {
2414         type => 'string',
2415         pattern => '[A-Za-z0-9+\/]+={0,2}',
2416         format_description => 'Base64 encoded string',
2417         description => "Set SMBIOS1 family string.",
2418         optional => 1,
2419     },
2420     base64 => {
2421         type => 'boolean',
2422         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2423         optional => 1,
2424     },
2425 };
2426
2427 sub parse_smbios1 {
2428     my ($data) = @_;
2429
2430     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2431     warn $@ if $@;
2432     return $res;
2433 }
2434
2435 sub print_smbios1 {
2436     my ($smbios1) = @_;
2437     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2438 }
2439
2440 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2441
2442 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2443 sub verify_bootdisk {
2444     my ($value, $noerr) = @_;
2445
2446     return $value if is_valid_drivename($value);
2447
2448     return undef if $noerr;
2449
2450     die "invalid boot disk '$value'\n";
2451 }
2452
2453 sub parse_watchdog {
2454     my ($value) = @_;
2455
2456     return undef if !$value;
2457
2458     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2459     warn $@ if $@;
2460     return $res;
2461 }
2462
2463 sub parse_guest_agent {
2464     my ($value) = @_;
2465
2466     return {} if !defined($value->{agent});
2467
2468     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2469     warn $@ if $@;
2470
2471     # if the agent is disabled ignore the other potentially set properties
2472     return {} if !$res->{enabled};
2473     return $res;
2474 }
2475
2476 sub parse_vga {
2477     my ($value) = @_;
2478
2479     return {} if !$value;
2480     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2481     warn $@ if $@;
2482     return $res;
2483 }
2484
2485 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2486 sub verify_usb_device {
2487     my ($value, $noerr) = @_;
2488
2489     return $value if parse_usb_device($value);
2490
2491     return undef if $noerr;
2492
2493     die "unable to parse usb device\n";
2494 }
2495
2496 # add JSON properties for create and set function
2497 sub json_config_properties {
2498     my $prop = shift;
2499
2500     foreach my $opt (keys %$confdesc) {
2501         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2502         $prop->{$opt} = $confdesc->{$opt};
2503     }
2504
2505     return $prop;
2506 }
2507
2508 # return copy of $confdesc_cloudinit to generate documentation
2509 sub cloudinit_config_properties {
2510
2511     return dclone($confdesc_cloudinit);
2512 }
2513
2514 sub check_type {
2515     my ($key, $value) = @_;
2516
2517     die "unknown setting '$key'\n" if !$confdesc->{$key};
2518
2519     my $type = $confdesc->{$key}->{type};
2520
2521     if (!defined($value)) {
2522         die "got undefined value\n";
2523     }
2524
2525     if ($value =~ m/[\n\r]/) {
2526         die "property contains a line feed\n";
2527     }
2528
2529     if ($type eq 'boolean') {
2530         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2531         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2532         die "type check ('boolean') failed - got '$value'\n";
2533     } elsif ($type eq 'integer') {
2534         return int($1) if $value =~ m/^(\d+)$/;
2535         die "type check ('integer') failed - got '$value'\n";
2536     } elsif ($type eq 'number') {
2537         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2538         die "type check ('number') failed - got '$value'\n";
2539     } elsif ($type eq 'string') {
2540         if (my $fmt = $confdesc->{$key}->{format}) {
2541             PVE::JSONSchema::check_format($fmt, $value);
2542             return $value;
2543         }
2544         $value =~ s/^\"(.*)\"$/$1/;
2545         return $value;
2546     } else {
2547         die "internal error"
2548     }
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conf = PVE::QemuConfig->load_config($vmid);
2555
2556     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2557
2558     if ($conf->{template}) {
2559         # check if any base image is still used by a linked clone
2560         foreach_drive($conf, sub {
2561                 my ($ds, $drive) = @_;
2562
2563                 return if drive_is_cdrom($drive);
2564
2565                 my $volid = $drive->{file};
2566
2567                 return if !$volid || $volid =~ m|^/|;
2568
2569                 die "base volume '$volid' is still in use by linked cloned\n"
2570                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2571
2572         });
2573     }
2574
2575     # only remove disks owned by this VM
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578
2579         return if drive_is_cdrom($drive, 1);
2580
2581         my $volid = $drive->{file};
2582
2583         return if !$volid || $volid =~ m|^/|;
2584
2585         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2586         return if !$path || !$owner || ($owner != $vmid);
2587
2588         eval {
2589             PVE::Storage::vdisk_free($storecfg, $volid);
2590         };
2591         warn "Could not remove disk '$volid', check manually: $@" if $@;
2592
2593     });
2594
2595     # also remove unused disk
2596     eval {
2597         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2598
2599         eval {
2600             PVE::Storage::foreach_volid($dl, sub {
2601                 my ($volid, $sid, $volname, $d) = @_;
2602                 PVE::Storage::vdisk_free($storecfg, $volid);
2603             });
2604         };
2605         warn $@ if $@;
2606
2607     };
2608     warn $@ if $@;
2609
2610     if ($keep_empty_config) {
2611         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2612     } else {
2613         PVE::QemuConfig->destroy_config($vmid);
2614     }
2615 }
2616
2617 sub parse_vm_config {
2618     my ($filename, $raw) = @_;
2619
2620     return undef if !defined($raw);
2621
2622     my $res = {
2623         digest => Digest::SHA::sha1_hex($raw),
2624         snapshots => {},
2625         pending => {},
2626     };
2627
2628     $filename =~ m|/qemu-server/(\d+)\.conf$|
2629         || die "got strange filename '$filename'";
2630
2631     my $vmid = $1;
2632
2633     my $conf = $res;
2634     my $descr;
2635     my $section = '';
2636
2637     my @lines = split(/\n/, $raw);
2638     foreach my $line (@lines) {
2639         next if $line =~ m/^\s*$/;
2640
2641         if ($line =~ m/^\[PENDING\]\s*$/i) {
2642             $section = 'pending';
2643             if (defined($descr)) {
2644                 $descr =~ s/\s+$//;
2645                 $conf->{description} = $descr;
2646             }
2647             $descr = undef;
2648             $conf = $res->{$section} = {};
2649             next;
2650
2651         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2652             $section = $1;
2653             if (defined($descr)) {
2654                 $descr =~ s/\s+$//;
2655                 $conf->{description} = $descr;
2656             }
2657             $descr = undef;
2658             $conf = $res->{snapshots}->{$section} = {};
2659             next;
2660         }
2661
2662         if ($line =~ m/^\#(.*)\s*$/) {
2663             $descr = '' if !defined($descr);
2664             $descr .= PVE::Tools::decode_text($1) . "\n";
2665             next;
2666         }
2667
2668         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2669             $descr = '' if !defined($descr);
2670             $descr .= PVE::Tools::decode_text($2);
2671         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2672             $conf->{snapstate} = $1;
2673         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2674             my $key = $1;
2675             my $value = $2;
2676             $conf->{$key} = $value;
2677         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2678             my $value = $1;
2679             if ($section eq 'pending') {
2680                 $conf->{delete} = $value; # we parse this later
2681             } else {
2682                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2683             }
2684         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2685             my $key = $1;
2686             my $value = $2;
2687             eval { $value = check_type($key, $value); };
2688             if ($@) {
2689                 warn "vm $vmid - unable to parse value of '$key' - $@";
2690             } else {
2691                 $key = 'ide2' if $key eq 'cdrom';
2692                 my $fmt = $confdesc->{$key}->{format};
2693                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2694                     my $v = parse_drive($key, $value);
2695                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2696                         $v->{file} = $volid;
2697                         $value = print_drive($vmid, $v);
2698                     } else {
2699                         warn "vm $vmid - unable to parse value of '$key'\n";
2700                         next;
2701                     }
2702                 }
2703
2704                 $conf->{$key} = $value;
2705             }
2706         }
2707     }
2708
2709     if (defined($descr)) {
2710         $descr =~ s/\s+$//;
2711         $conf->{description} = $descr;
2712     }
2713     delete $res->{snapstate}; # just to be sure
2714
2715     return $res;
2716 }
2717
2718 sub write_vm_config {
2719     my ($filename, $conf) = @_;
2720
2721     delete $conf->{snapstate}; # just to be sure
2722
2723     if ($conf->{cdrom}) {
2724         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2725         $conf->{ide2} = $conf->{cdrom};
2726         delete $conf->{cdrom};
2727     }
2728
2729     # we do not use 'smp' any longer
2730     if ($conf->{sockets}) {
2731         delete $conf->{smp};
2732     } elsif ($conf->{smp}) {
2733         $conf->{sockets} = $conf->{smp};
2734         delete $conf->{cores};
2735         delete $conf->{smp};
2736     }
2737
2738     my $used_volids = {};
2739
2740     my $cleanup_config = sub {
2741         my ($cref, $pending, $snapname) = @_;
2742
2743         foreach my $key (keys %$cref) {
2744             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2745                 $key eq 'snapstate' || $key eq 'pending';
2746             my $value = $cref->{$key};
2747             if ($key eq 'delete') {
2748                 die "propertry 'delete' is only allowed in [PENDING]\n"
2749                     if !$pending;
2750                 # fixme: check syntax?
2751                 next;
2752             }
2753             eval { $value = check_type($key, $value); };
2754             die "unable to parse value of '$key' - $@" if $@;
2755
2756             $cref->{$key} = $value;
2757
2758             if (!$snapname && is_valid_drivename($key)) {
2759                 my $drive = parse_drive($key, $value);
2760                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2761             }
2762         }
2763     };
2764
2765     &$cleanup_config($conf);
2766
2767     &$cleanup_config($conf->{pending}, 1);
2768
2769     foreach my $snapname (keys %{$conf->{snapshots}}) {
2770         die "internal error" if $snapname eq 'pending';
2771         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2772     }
2773
2774     # remove 'unusedX' settings if we re-add a volume
2775     foreach my $key (keys %$conf) {
2776         my $value = $conf->{$key};
2777         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2778             delete $conf->{$key};
2779         }
2780     }
2781
2782     my $generate_raw_config = sub {
2783         my ($conf, $pending) = @_;
2784
2785         my $raw = '';
2786
2787         # add description as comment to top of file
2788         if (defined(my $descr = $conf->{description})) {
2789             if ($descr) {
2790                 foreach my $cl (split(/\n/, $descr)) {
2791                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2792                 }
2793             } else {
2794                 $raw .= "#\n" if $pending;
2795             }
2796         }
2797
2798         foreach my $key (sort keys %$conf) {
2799             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2800             $raw .= "$key: $conf->{$key}\n";
2801         }
2802         return $raw;
2803     };
2804
2805     my $raw = &$generate_raw_config($conf);
2806
2807     if (scalar(keys %{$conf->{pending}})){
2808         $raw .= "\n[PENDING]\n";
2809         $raw .= &$generate_raw_config($conf->{pending}, 1);
2810     }
2811
2812     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2813         $raw .= "\n[$snapname]\n";
2814         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2815     }
2816
2817     return $raw;
2818 }
2819
2820 sub load_defaults {
2821
2822     my $res = {};
2823
2824     # we use static defaults from our JSON schema configuration
2825     foreach my $key (keys %$confdesc) {
2826         if (defined(my $default = $confdesc->{$key}->{default})) {
2827             $res->{$key} = $default;
2828         }
2829     }
2830
2831     return $res;
2832 }
2833
2834 sub config_list {
2835     my $vmlist = PVE::Cluster::get_vmlist();
2836     my $res = {};
2837     return $res if !$vmlist || !$vmlist->{ids};
2838     my $ids = $vmlist->{ids};
2839
2840     foreach my $vmid (keys %$ids) {
2841         my $d = $ids->{$vmid};
2842         next if !$d->{node} || $d->{node} ne $nodename;
2843         next if !$d->{type} || $d->{type} ne 'qemu';
2844         $res->{$vmid}->{exists} = 1;
2845     }
2846     return $res;
2847 }
2848
2849 # test if VM uses local resources (to prevent migration)
2850 sub check_local_resources {
2851     my ($conf, $noerr) = @_;
2852
2853     my @loc_res = ();
2854
2855     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2856     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2857
2858     push @loc_res, "ivshmem" if $conf->{ivshmem};
2859
2860     foreach my $k (keys %$conf) {
2861         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2862         # sockets are safe: they will recreated be on the target side post-migrate
2863         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2864         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2865     }
2866
2867     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2868
2869     return \@loc_res;
2870 }
2871
2872 # check if used storages are available on all nodes (use by migrate)
2873 sub check_storage_availability {
2874     my ($storecfg, $conf, $node) = @_;
2875
2876     foreach_drive($conf, sub {
2877         my ($ds, $drive) = @_;
2878
2879         my $volid = $drive->{file};
2880         return if !$volid;
2881
2882         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2883         return if !$sid;
2884
2885         # check if storage is available on both nodes
2886         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2887         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2888    });
2889 }
2890
2891 # list nodes where all VM images are available (used by has_feature API)
2892 sub shared_nodes {
2893     my ($conf, $storecfg) = @_;
2894
2895     my $nodelist = PVE::Cluster::get_nodelist();
2896     my $nodehash = { map { $_ => 1 } @$nodelist };
2897     my $nodename = PVE::INotify::nodename();
2898
2899     foreach_drive($conf, sub {
2900         my ($ds, $drive) = @_;
2901
2902         my $volid = $drive->{file};
2903         return if !$volid;
2904
2905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906         if ($storeid) {
2907             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2908             if ($scfg->{disable}) {
2909                 $nodehash = {};
2910             } elsif (my $avail = $scfg->{nodes}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     delete $nodehash->{$node} if !$avail->{$node};
2913                 }
2914             } elsif (!$scfg->{shared}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     delete $nodehash->{$node} if $node ne $nodename
2917                 }
2918             }
2919         }
2920     });
2921
2922     return $nodehash
2923 }
2924
2925 sub check_local_storage_availability {
2926     my ($conf, $storecfg) = @_;
2927
2928     my $nodelist = PVE::Cluster::get_nodelist();
2929     my $nodehash = { map { $_ => {} } @$nodelist };
2930
2931     foreach_drive($conf, sub {
2932         my ($ds, $drive) = @_;
2933
2934         my $volid = $drive->{file};
2935         return if !$volid;
2936
2937         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2938         if ($storeid) {
2939             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2940
2941             if ($scfg->{disable}) {
2942                 foreach my $node (keys %$nodehash) {
2943                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2944                 }
2945             } elsif (my $avail = $scfg->{nodes}) {
2946                 foreach my $node (keys %$nodehash) {
2947                     if (!$avail->{$node}) {
2948                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2949                     }
2950                 }
2951             }
2952         }
2953     });
2954
2955     foreach my $node (values %$nodehash) {
2956         if (my $unavail = $node->{unavailable_storages}) {
2957             $node->{unavailable_storages} = [ sort keys %$unavail ];
2958         }
2959     }
2960
2961     return $nodehash
2962 }
2963
2964 sub check_cmdline {
2965     my ($pidfile, $pid) = @_;
2966
2967     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2968     if (defined($fh)) {
2969         my $line = <$fh>;
2970         $fh->close;
2971         return undef if !$line;
2972         my @param = split(/\0/, $line);
2973
2974         my $cmd = $param[0];
2975         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2976
2977         for (my $i = 0; $i < scalar (@param); $i++) {
2978             my $p = $param[$i];
2979             next if !$p;
2980             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2981                 my $p = $param[$i+1];
2982                 return 1 if $p && ($p eq $pidfile);
2983                 return undef;
2984             }
2985         }
2986     }
2987     return undef;
2988 }
2989
2990 sub check_running {
2991     my ($vmid, $nocheck, $node) = @_;
2992
2993     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2994
2995     die "unable to find configuration file for VM $vmid - no such machine\n"
2996         if !$nocheck && ! -f $filename;
2997
2998     my $pidfile = pidfile_name($vmid);
2999
3000     if (my $fd = IO::File->new("<$pidfile")) {
3001         my $st = stat($fd);
3002         my $line = <$fd>;
3003         close($fd);
3004
3005         my $mtime = $st->mtime;
3006         if ($mtime > time()) {
3007             warn "file '$filename' modified in future\n";
3008         }
3009
3010         if ($line =~ m/^(\d+)$/) {
3011             my $pid = $1;
3012             if (check_cmdline($pidfile, $pid)) {
3013                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3014                     return $pid;
3015                 }
3016             }
3017         }
3018     }
3019
3020     return undef;
3021 }
3022
3023 sub vzlist {
3024
3025     my $vzlist = config_list();
3026
3027     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3028
3029     while (defined(my $de = $fd->read)) {
3030         next if $de !~ m/^(\d+)\.pid$/;
3031         my $vmid = $1;
3032         next if !defined($vzlist->{$vmid});
3033         if (my $pid = check_running($vmid)) {
3034             $vzlist->{$vmid}->{pid} = $pid;
3035         }
3036     }
3037
3038     return $vzlist;
3039 }
3040
3041 sub disksize {
3042     my ($storecfg, $conf) = @_;
3043
3044     my $bootdisk = $conf->{bootdisk};
3045     return undef if !$bootdisk;
3046     return undef if !is_valid_drivename($bootdisk);
3047
3048     return undef if !$conf->{$bootdisk};
3049
3050     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3051     return undef if !defined($drive);
3052
3053     return undef if drive_is_cdrom($drive);
3054
3055     my $volid = $drive->{file};
3056     return undef if !$volid;
3057
3058     return $drive->{size};
3059 }
3060
3061 our $vmstatus_return_properties = {
3062     vmid => get_standard_option('pve-vmid'),
3063     status => {
3064         description => "Qemu process status.",
3065         type => 'string',
3066         enum => ['stopped', 'running'],
3067     },
3068     maxmem => {
3069         description => "Maximum memory in bytes.",
3070         type => 'integer',
3071         optional => 1,
3072         renderer => 'bytes',
3073     },
3074     maxdisk => {
3075         description => "Root disk size in bytes.",
3076         type => 'integer',
3077         optional => 1,
3078         renderer => 'bytes',
3079     },
3080     name => {
3081         description => "VM name.",
3082         type => 'string',
3083         optional => 1,
3084     },
3085     qmpstatus => {
3086         description => "Qemu QMP agent status.",
3087         type => 'string',
3088         optional => 1,
3089     },
3090     pid => {
3091         description => "PID of running qemu process.",
3092         type => 'integer',
3093         optional => 1,
3094     },
3095     uptime => {
3096         description => "Uptime.",
3097         type => 'integer',
3098         optional => 1,
3099         renderer => 'duration',
3100     },
3101     cpus => {
3102         description => "Maximum usable CPUs.",
3103         type => 'number',
3104         optional => 1,
3105     },
3106     lock => {
3107         description => "The current config lock, if any.",
3108         type => 'string',
3109         optional => 1,
3110     }
3111 };
3112
3113 my $last_proc_pid_stat;
3114
3115 # get VM status information
3116 # This must be fast and should not block ($full == false)
3117 # We only query KVM using QMP if $full == true (this can be slow)
3118 sub vmstatus {
3119     my ($opt_vmid, $full) = @_;
3120
3121     my $res = {};
3122
3123     my $storecfg = PVE::Storage::config();
3124
3125     my $list = vzlist();
3126     my $defaults = load_defaults();
3127
3128     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3129
3130     my $cpucount = $cpuinfo->{cpus} || 1;
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134
3135         my $conf = PVE::QemuConfig->load_config($vmid);
3136
3137         my $d = { vmid => $vmid };
3138         $d->{pid} = $list->{$vmid}->{pid};
3139
3140         # fixme: better status?
3141         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3142
3143         my $size = disksize($storecfg, $conf);
3144         if (defined($size)) {
3145             $d->{disk} = 0; # no info available
3146             $d->{maxdisk} = $size;
3147         } else {
3148             $d->{disk} = 0;
3149             $d->{maxdisk} = 0;
3150         }
3151
3152         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3153             * ($conf->{cores} || $defaults->{cores});
3154         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3155         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3156
3157         $d->{name} = $conf->{name} || "VM $vmid";
3158         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3159             : $defaults->{memory}*(1024*1024);
3160
3161         if ($conf->{balloon}) {
3162             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3163             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3164                 : $defaults->{shares};
3165         }
3166
3167         $d->{uptime} = 0;
3168         $d->{cpu} = 0;
3169         $d->{mem} = 0;
3170
3171         $d->{netout} = 0;
3172         $d->{netin} = 0;
3173
3174         $d->{diskread} = 0;
3175         $d->{diskwrite} = 0;
3176
3177         $d->{template} = PVE::QemuConfig->is_template($conf);
3178
3179         $d->{serial} = 1 if conf_has_serial($conf);
3180         $d->{lock} = $conf->{lock} if $conf->{lock};
3181
3182         $res->{$vmid} = $d;
3183     }
3184
3185     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3186     foreach my $dev (keys %$netdev) {
3187         next if $dev !~ m/^tap([1-9]\d*)i/;
3188         my $vmid = $1;
3189         my $d = $res->{$vmid};
3190         next if !$d;
3191
3192         $d->{netout} += $netdev->{$dev}->{receive};
3193         $d->{netin} += $netdev->{$dev}->{transmit};
3194
3195         if ($full) {
3196             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3197             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3198         }
3199
3200     }
3201
3202     my $ctime = gettimeofday;
3203
3204     foreach my $vmid (keys %$list) {
3205
3206         my $d = $res->{$vmid};
3207         my $pid = $d->{pid};
3208         next if !$pid;
3209
3210         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3211         next if !$pstat; # not running
3212
3213         my $used = $pstat->{utime} + $pstat->{stime};
3214
3215         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3216
3217         if ($pstat->{vsize}) {
3218             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3219         }
3220
3221         my $old = $last_proc_pid_stat->{$pid};
3222         if (!$old) {
3223             $last_proc_pid_stat->{$pid} = {
3224                 time => $ctime,
3225                 used => $used,
3226                 cpu => 0,
3227             };
3228             next;
3229         }
3230
3231         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3232
3233         if ($dtime > 1000) {
3234             my $dutime = $used -  $old->{used};
3235
3236             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3237             $last_proc_pid_stat->{$pid} = {
3238                 time => $ctime,
3239                 used => $used,
3240                 cpu => $d->{cpu},
3241             };
3242         } else {
3243             $d->{cpu} = $old->{cpu};
3244         }
3245     }
3246
3247     return $res if !$full;
3248
3249     my $qmpclient = PVE::QMPClient->new();
3250
3251     my $ballooncb = sub {
3252         my ($vmid, $resp) = @_;
3253
3254         my $info = $resp->{'return'};
3255         return if !$info->{max_mem};
3256
3257         my $d = $res->{$vmid};
3258
3259         # use memory assigned to VM
3260         $d->{maxmem} = $info->{max_mem};
3261         $d->{balloon} = $info->{actual};
3262
3263         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3264             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3265             $d->{freemem} = $info->{free_mem};
3266         }
3267
3268         $d->{ballooninfo} = $info;
3269     };
3270
3271     my $blockstatscb = sub {
3272         my ($vmid, $resp) = @_;
3273         my $data = $resp->{'return'} || [];
3274         my $totalrdbytes = 0;
3275         my $totalwrbytes = 0;
3276
3277         for my $blockstat (@$data) {
3278             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3279             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3280
3281             $blockstat->{device} =~ s/drive-//;
3282             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3283         }
3284         $res->{$vmid}->{diskread} = $totalrdbytes;
3285         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3286     };
3287
3288     my $statuscb = sub {
3289         my ($vmid, $resp) = @_;
3290
3291         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3292         # this fails if ballon driver is not loaded, so this must be
3293         # the last commnand (following command are aborted if this fails).
3294         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3295
3296         my $status = 'unknown';
3297         if (!defined($status = $resp->{'return'}->{status})) {
3298             warn "unable to get VM status\n";
3299             return;
3300         }
3301
3302         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3303     };
3304
3305     foreach my $vmid (keys %$list) {
3306         next if $opt_vmid && ($vmid ne $opt_vmid);
3307         next if !$res->{$vmid}->{pid}; # not running
3308         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3309     }
3310
3311     $qmpclient->queue_execute(undef, 2);
3312
3313     foreach my $vmid (keys %$list) {
3314         next if $opt_vmid && ($vmid ne $opt_vmid);
3315         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3316     }
3317
3318     return $res;
3319 }
3320
3321 sub foreach_drive {
3322     my ($conf, $func, @param) = @_;
3323
3324     foreach my $ds (valid_drive_names()) {
3325         next if !defined($conf->{$ds});
3326
3327         my $drive = parse_drive($ds, $conf->{$ds});
3328         next if !$drive;
3329
3330         &$func($ds, $drive, @param);
3331     }
3332 }
3333
3334 sub foreach_volid {
3335     my ($conf, $func, @param) = @_;
3336
3337     my $volhash = {};
3338
3339     my $test_volid = sub {
3340         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3341
3342         return if !$volid;
3343
3344         $volhash->{$volid}->{cdrom} //= 1;
3345         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3346
3347         $volhash->{$volid}->{replicate} //= 0;
3348         $volhash->{$volid}->{replicate} = 1 if $replicate;
3349
3350         $volhash->{$volid}->{shared} //= 0;
3351         $volhash->{$volid}->{shared} = 1 if $shared;
3352
3353         $volhash->{$volid}->{referenced_in_config} //= 0;
3354         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3355
3356         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3357             if defined($snapname);
3358         $volhash->{$volid}->{size} = $size if $size;
3359     };
3360
3361     foreach_drive($conf, sub {
3362         my ($ds, $drive) = @_;
3363         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3364     });
3365
3366     foreach my $snapname (keys %{$conf->{snapshots}}) {
3367         my $snap = $conf->{snapshots}->{$snapname};
3368         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3369         foreach_drive($snap, sub {
3370             my ($ds, $drive) = @_;
3371             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3372         });
3373     }
3374
3375     foreach my $volid (keys %$volhash) {
3376         &$func($volid, $volhash->{$volid}, @param);
3377     }
3378 }
3379
3380 sub conf_has_serial {
3381     my ($conf) = @_;
3382
3383     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3384         if ($conf->{"serial$i"}) {
3385             return 1;
3386         }
3387     }
3388
3389     return 0;
3390 }
3391
3392 sub conf_has_audio {
3393     my ($conf, $id) = @_;
3394
3395     $id //= 0;
3396     my $audio = $conf->{"audio$id"};
3397     return undef if !defined($audio);
3398
3399     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3400     my $audiodriver = $audioproperties->{driver} // 'spice';
3401
3402     return {
3403         dev => $audioproperties->{device},
3404         dev_id => "audiodev$id",
3405         backend => $audiodriver,
3406         backend_id => "$audiodriver-backend${id}",
3407     };
3408 }
3409
3410 sub vga_conf_has_spice {
3411     my ($vga) = @_;
3412
3413     my $vgaconf = parse_vga($vga);
3414     my $vgatype = $vgaconf->{type};
3415     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3416
3417     return $1 || 1;
3418 }
3419
3420 sub is_native($) {
3421     my ($arch) = @_;
3422     return get_host_arch() eq $arch;
3423 }
3424
3425 my $default_machines = {
3426     x86_64 => 'pc',
3427     aarch64 => 'virt',
3428 };
3429
3430 sub get_basic_machine_info {
3431     my ($conf, $forcemachine) = @_;
3432
3433     my $arch = $conf->{arch} // get_host_arch();
3434     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3435     return ($arch, $machine);
3436 }
3437
3438 sub get_ovmf_files($) {
3439     my ($arch) = @_;
3440
3441     my $ovmf = $OVMF->{$arch}
3442         or die "no OVMF images known for architecture '$arch'\n";
3443
3444     return @$ovmf;
3445 }
3446
3447 my $Arch2Qemu = {
3448     aarch64 => '/usr/bin/qemu-system-aarch64',
3449     x86_64 => '/usr/bin/qemu-system-x86_64',
3450 };
3451 sub get_command_for_arch($) {
3452     my ($arch) = @_;
3453     return '/usr/bin/kvm' if is_native($arch);
3454
3455     my $cmd = $Arch2Qemu->{$arch}
3456         or die "don't know how to emulate architecture '$arch'\n";
3457     return $cmd;
3458 }
3459
3460 sub get_cpu_options {
3461     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3462
3463     my $cpuFlags = [];
3464     my $ostype = $conf->{ostype};
3465
3466     my $cpu = $kvm ? "kvm64" : "qemu64";
3467     if ($arch eq 'aarch64') {
3468         $cpu = 'cortex-a57';
3469     }
3470     my $hv_vendor_id;
3471     if (my $cputype = $conf->{cpu}) {
3472         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3473             or die "Cannot parse cpu description: $cputype\n";
3474         $cpu = $cpuconf->{cputype};
3475         $kvm_off = 1 if $cpuconf->{hidden};
3476         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3477
3478         if (defined(my $flags = $cpuconf->{flags})) {
3479             push @$cpuFlags, split(";", $flags);
3480         }
3481     }
3482
3483     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3484
3485     push @$cpuFlags , '-x2apic'
3486         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3487
3488     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3489
3490     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3491
3492     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3493
3494         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3495         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3496     }
3497
3498     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3499
3500     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3501
3502     push @$cpuFlags, 'kvm=off' if $kvm_off;
3503
3504     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3505         push @$cpuFlags, "vendor=${cpu_vendor}"
3506             if $cpu_vendor ne 'default';
3507     } elsif ($arch ne 'aarch64') {
3508         die "internal error"; # should not happen
3509     }
3510
3511     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3512
3513     return ('-cpu', $cpu);
3514 }
3515
3516 sub config_to_command {
3517     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3518
3519     my $cmd = [];
3520     my $globalFlags = [];
3521     my $machineFlags = [];
3522     my $rtcFlags = [];
3523     my $devices = [];
3524     my $pciaddr = '';
3525     my $bridges = {};
3526     my $vernum = 0; # unknown
3527     my $ostype = $conf->{ostype};
3528     my $winversion = windows_version($ostype);
3529     my $kvm = $conf->{kvm};
3530
3531     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3532     my $kvm_binary = get_command_for_arch($arch);
3533     my $kvmver = kvm_user_version($kvm_binary);
3534     $kvm //= 1 if is_native($arch);
3535
3536     if ($kvm) {
3537         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3538             if !defined kvm_version();
3539     }
3540
3541     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3542         $vernum = $1*1000000+$2*1000;
3543     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3544         $vernum = $1*1000000+$2*1000+$3;
3545     }
3546
3547     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3548
3549     my $q35 = machine_type_is_q35($conf);
3550     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3551     my $use_old_bios_files = undef;
3552     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3553
3554     my $cpuunits = defined($conf->{cpuunits}) ?
3555             $conf->{cpuunits} : $defaults->{cpuunits};
3556
3557     push @$cmd, $kvm_binary;
3558
3559     push @$cmd, '-id', $vmid;
3560
3561     my $vmname = $conf->{name} || "vm$vmid";
3562
3563     push @$cmd, '-name', $vmname;
3564
3565     my $use_virtio = 0;
3566
3567     my $qmpsocket = qmp_socket($vmid);
3568     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3569     push @$cmd, '-mon', "chardev=qmp,mode=control";
3570
3571     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3572         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3573         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3574     }
3575
3576     push @$cmd, '-pidfile' , pidfile_name($vmid);
3577
3578     push @$cmd, '-daemonize';
3579
3580     if ($conf->{smbios1}) {
3581         my $smbios_conf = parse_smbios1($conf->{smbios1});
3582         if ($smbios_conf->{base64}) {
3583             # Do not pass base64 flag to qemu
3584             delete $smbios_conf->{base64};
3585             my $smbios_string = "";
3586             foreach my $key (keys %$smbios_conf) {
3587                 my $value;
3588                 if ($key eq "uuid") {
3589                     $value = $smbios_conf->{uuid}
3590                 } else {
3591                     $value = decode_base64($smbios_conf->{$key});
3592                 }
3593                 # qemu accepts any binary data, only commas need escaping by double comma
3594                 $value =~ s/,/,,/g;
3595                 $smbios_string .= "," . $key . "=" . $value if $value;
3596             }
3597             push @$cmd, '-smbios', "type=1" . $smbios_string;
3598         } else {
3599             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3600         }
3601     }
3602
3603     if ($conf->{vmgenid}) {
3604         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3605     }
3606
3607     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3608     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3609         die "uefi base image not found\n" if ! -f $ovmf_code;
3610
3611         my $path;
3612         my $format;
3613         if (my $efidisk = $conf->{efidisk0}) {
3614             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3615             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3616             $format = $d->{format};
3617             if ($storeid) {
3618                 $path = PVE::Storage::path($storecfg, $d->{file});
3619                 if (!defined($format)) {
3620                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3621                     $format = qemu_img_format($scfg, $volname);
3622                 }
3623             } else {
3624                 $path = $d->{file};
3625                 die "efidisk format must be specified\n"
3626                     if !defined($format);
3627             }
3628         } else {
3629             warn "no efidisk configured! Using temporary efivars disk.\n";
3630             $path = "/tmp/$vmid-ovmf.fd";
3631             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3632             $format = 'raw';
3633         }
3634
3635         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3636         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3637     }
3638
3639     # load q35 config
3640     if ($q35) {
3641         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3642         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3644         } else {
3645             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3646         }
3647     }
3648
3649     # add usb controllers
3650     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3651     push @$devices, @usbcontrollers if @usbcontrollers;
3652     my $vga = parse_vga($conf->{vga});
3653
3654     my $qxlnum = vga_conf_has_spice($conf->{vga});
3655     $vga->{type} = 'qxl' if $qxlnum;
3656
3657     if (!$vga->{type}) {
3658         if ($arch eq 'aarch64') {
3659             $vga->{type} = 'virtio';
3660         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3661             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3662         } else {
3663             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3664         }
3665     }
3666
3667     # enable absolute mouse coordinates (needed by vnc)
3668     my $tablet;
3669     if (defined($conf->{tablet})) {
3670         $tablet = $conf->{tablet};
3671     } else {
3672         $tablet = $defaults->{tablet};
3673         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3674         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3675     }
3676
3677     if ($tablet) {
3678         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3679         my $kbd = print_keyboarddevice_full($conf, $arch);
3680         push @$devices, '-device', $kbd if defined($kbd);
3681     }
3682
3683     my $kvm_off = 0;
3684     my $gpu_passthrough;
3685
3686     # host pci devices
3687     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3688         my $id = "hostpci$i";
3689         my $d = parse_hostpci($conf->{$id});
3690         next if !$d;
3691
3692         if (my $pcie = $d->{pcie}) {
3693             die "q35 machine model is not enabled" if !$q35;
3694             # win7 wants to have the pcie devices directly on the pcie bus
3695             # instead of in the root port
3696             if ($winversion == 7) {
3697                 $pciaddr = print_pcie_addr("${id}bus0");
3698             } else {
3699                 # add more root ports if needed, 4 are present by default
3700                 # by pve-q35 cfgs, rest added here on demand.
3701                 if ($i > 3) {
3702                     push @$devices, '-device', print_pcie_root_port($i);
3703                 }
3704                 $pciaddr = print_pcie_addr($id);
3705             }
3706         } else {
3707             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3708         }
3709
3710         my $xvga = '';
3711         if ($d->{'x-vga'}) {
3712             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3713             $kvm_off = 1;
3714             $vga->{type} = 'none' if !defined($conf->{vga});
3715             $gpu_passthrough = 1;
3716         }
3717
3718         my $pcidevices = $d->{pciid};
3719         my $multifunction = 1 if @$pcidevices > 1;
3720
3721         my $sysfspath;
3722         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3723             my $pci_id = $pcidevices->[0]->{id};
3724             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3725             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3726         } elsif ($d->{mdev}) {
3727             warn "ignoring mediated device '$id' with multifunction device\n";
3728         }
3729
3730         my $j=0;
3731         foreach my $pcidevice (@$pcidevices) {
3732             my $devicestr = "vfio-pci";
3733
3734             if ($sysfspath) {
3735                 $devicestr .= ",sysfsdev=$sysfspath";
3736             } else {
3737                 $devicestr .= ",host=$pcidevice->{id}";
3738             }
3739
3740             my $mf_addr = $multifunction ? ".$j" : '';
3741             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3742
3743             if ($j == 0) {
3744                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3745                 $devicestr .= "$xvga";
3746                 $devicestr .= ",multifunction=on" if $multifunction;
3747                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3748             }
3749
3750             push @$devices, '-device', $devicestr;
3751             $j++;
3752         }
3753     }
3754
3755     # usb devices
3756     my $usb_dev_features = {};
3757     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3758
3759     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3760     push @$devices, @usbdevices if @usbdevices;
3761     # serial devices
3762     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3763         if (my $path = $conf->{"serial$i"}) {
3764             if ($path eq 'socket') {
3765                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3766                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3767                 # On aarch64, serial0 is the UART device. Qemu only allows
3768                 # connecting UART devices via the '-serial' command line, as
3769                 # the device has a fixed slot on the hardware...
3770                 if ($arch eq 'aarch64' && $i == 0) {
3771                     push @$devices, '-serial', "chardev:serial$i";
3772                 } else {
3773                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3774                 }
3775             } else {
3776                 die "no such serial device\n" if ! -c $path;
3777                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3778                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3779             }
3780         }
3781     }
3782
3783     # parallel devices
3784     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3785         if (my $path = $conf->{"parallel$i"}) {
3786             die "no such parallel device\n" if ! -c $path;
3787             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3788             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3789             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3790         }
3791     }
3792
3793     if (my $audio = conf_has_audio($conf)) {
3794
3795         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3796
3797         my $id = $audio->{dev_id};
3798         if ($audio->{dev} eq 'AC97') {
3799             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3800         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3801             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3802             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3803             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3804         } else {
3805             die "unkown audio device '$audio->{dev}', implement me!";
3806         }
3807
3808         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3809     }
3810
3811     my $sockets = 1;
3812     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3813     $sockets = $conf->{sockets} if  $conf->{sockets};
3814
3815     my $cores = $conf->{cores} || 1;
3816
3817     my $maxcpus = $sockets * $cores;
3818
3819     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3820
3821     my $allowed_vcpus = $cpuinfo->{cpus};
3822
3823     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3824         if ($allowed_vcpus < $maxcpus);
3825
3826     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3827
3828         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3829         for (my $i = 2; $i <= $vcpus; $i++)  {
3830             my $cpustr = print_cpu_device($conf,$i);
3831             push @$cmd, '-device', $cpustr;
3832         }
3833
3834     } else {
3835
3836         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3837     }
3838     push @$cmd, '-nodefaults';
3839
3840     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3841
3842     my $bootindex_hash = {};
3843     my $i = 1;
3844     foreach my $o (split(//, $bootorder)) {
3845         $bootindex_hash->{$o} = $i*100;
3846         $i++;
3847     }
3848
3849     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3850
3851     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3852
3853     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3854
3855     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3856         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3857         my $socket = vnc_socket($vmid);
3858         push @$cmd,  '-vnc', "unix:$socket,password";
3859     } else {
3860         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3861         push @$cmd, '-nographic';
3862     }
3863
3864     # time drift fix
3865     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3866
3867     my $useLocaltime = $conf->{localtime};
3868
3869     if ($winversion >= 5) { # windows
3870         $useLocaltime = 1 if !defined($conf->{localtime});
3871
3872         # use time drift fix when acpi is enabled
3873         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3874             $tdf = 1 if !defined($conf->{tdf});
3875         }
3876     }
3877
3878     if ($winversion >= 6) {
3879         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3880         push @$cmd, '-no-hpet';
3881     }
3882
3883     push @$rtcFlags, 'driftfix=slew' if $tdf;
3884
3885     if (!$kvm) {
3886         push @$machineFlags, 'accel=tcg';
3887     }
3888
3889     if ($machine_type) {
3890         push @$machineFlags, "type=${machine_type}";
3891     }
3892
3893     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3894         push @$rtcFlags, "base=$conf->{startdate}";
3895     } elsif ($useLocaltime) {
3896         push @$rtcFlags, 'base=localtime';
3897     }
3898
3899     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3900
3901     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3902
3903     push @$cmd, '-S' if $conf->{freeze};
3904
3905     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3906
3907     if (parse_guest_agent($conf)->{enabled}) {
3908         my $qgasocket = qmp_socket($vmid, 1);
3909         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3910         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3911         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3912         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3913     }
3914
3915     my $spice_port;
3916
3917     if ($qxlnum) {
3918         if ($qxlnum > 1) {
3919             if ($winversion){
3920                 for(my $i = 1; $i < $qxlnum; $i++){
3921                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3922                 }
3923             } else {
3924                 # assume other OS works like Linux
3925                 my ($ram, $vram) = ("134217728", "67108864");
3926                 if ($vga->{memory}) {
3927                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3928                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3929                 }
3930                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3931                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3932             }
3933         }
3934
3935         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3936
3937         my $nodename = PVE::INotify::nodename();
3938         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3939         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3940         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3941
3942         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3943         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3944         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3945
3946         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3947         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3948
3949         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3950         if ($spice_enhancement->{foldersharing}) {
3951             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3952             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3953         }
3954
3955         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3956         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3957         push @$devices, '-spice', "$spice_opts";
3958     }
3959
3960     # enable balloon by default, unless explicitly disabled
3961     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3962         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3963         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3964     }
3965
3966     if ($conf->{watchdog}) {
3967         my $wdopts = parse_watchdog($conf->{watchdog});
3968         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3969         my $watchdog = $wdopts->{model} || 'i6300esb';
3970         push @$devices, '-device', "$watchdog$pciaddr";
3971         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3972     }
3973
3974     my $vollist = [];
3975     my $scsicontroller = {};
3976     my $ahcicontroller = {};
3977     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3978
3979     # Add iscsi initiator name if available
3980     if (my $initiator = get_initiator_name()) {
3981         push @$devices, '-iscsi', "initiator-name=$initiator";
3982     }
3983
3984     foreach_drive($conf, sub {
3985         my ($ds, $drive) = @_;
3986
3987         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3988             push @$vollist, $drive->{file};
3989         }
3990
3991         # ignore efidisk here, already added in bios/fw handling code above
3992         return if $drive->{interface} eq 'efidisk';
3993
3994         $use_virtio = 1 if $ds =~ m/^virtio/;
3995
3996         if (drive_is_cdrom ($drive)) {
3997             if ($bootindex_hash->{d}) {
3998                 $drive->{bootindex} = $bootindex_hash->{d};
3999                 $bootindex_hash->{d} += 1;
4000             }
4001         } else {
4002             if ($bootindex_hash->{c}) {
4003                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4004                 $bootindex_hash->{c} += 1;
4005             }
4006         }
4007
4008         if($drive->{interface} eq 'virtio'){
4009            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4010         }
4011
4012         if ($drive->{interface} eq 'scsi') {
4013
4014             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4015
4016             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4017             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4018
4019             my $iothread = '';
4020             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4021                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4022                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4023             } elsif ($drive->{iothread}) {
4024                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4025             }
4026
4027             my $queues = '';
4028             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4029                 $queues = ",num_queues=$drive->{queues}";
4030             }
4031
4032             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4033             $scsicontroller->{$controller}=1;
4034         }
4035
4036         if ($drive->{interface} eq 'sata') {
4037            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4038            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4039            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4040            $ahcicontroller->{$controller}=1;
4041         }
4042
4043         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4044         push @$devices, '-drive',$drive_cmd;
4045         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4046     });
4047
4048     for (my $i = 0; $i < $MAX_NETS; $i++) {
4049          next if !$conf->{"net$i"};
4050          my $d = parse_net($conf->{"net$i"});
4051          next if !$d;
4052
4053          $use_virtio = 1 if $d->{model} eq 'virtio';
4054
4055          if ($bootindex_hash->{n}) {
4056             $d->{bootindex} = $bootindex_hash->{n};
4057             $bootindex_hash->{n} += 1;
4058          }
4059
4060          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4061          push @$devices, '-netdev', $netdevfull;
4062
4063          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4064          push @$devices, '-device', $netdevicefull;
4065     }
4066
4067     if ($conf->{ivshmem}) {
4068         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4069
4070         my $bus;
4071         if ($q35) {
4072             $bus = print_pcie_addr("ivshmem");
4073         } else {
4074             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4075         }
4076
4077         my $ivshmem_name = $ivshmem->{name} // $vmid;
4078         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4079
4080         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4081         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4082     }
4083
4084     if (!$q35) {
4085         # add pci bridges
4086         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4087            $bridges->{1} = 1;
4088            $bridges->{2} = 1;
4089         }
4090
4091         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4092
4093         for my $k (sort {$b cmp $a} keys %$bridges) {
4094             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4095             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4096         }
4097     }
4098
4099     push @$cmd, @$devices;
4100     push @$cmd, '-rtc', join(',', @$rtcFlags)
4101         if scalar(@$rtcFlags);
4102     push @$cmd, '-machine', join(',', @$machineFlags)
4103         if scalar(@$machineFlags);
4104     push @$cmd, '-global', join(',', @$globalFlags)
4105         if scalar(@$globalFlags);
4106
4107     if (my $vmstate = $conf->{vmstate}) {
4108         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4109         push @$vollist, $vmstate;
4110         push @$cmd, '-loadstate', $statepath;
4111     }
4112
4113     # add custom args
4114     if ($conf->{args}) {
4115         my $aa = PVE::Tools::split_args($conf->{args});
4116         push @$cmd, @$aa;
4117     }
4118
4119     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4120 }
4121
4122 sub vnc_socket {
4123     my ($vmid) = @_;
4124     return "${var_run_tmpdir}/$vmid.vnc";
4125 }
4126
4127 sub spice_port {
4128     my ($vmid) = @_;
4129
4130     my $res = vm_mon_cmd($vmid, 'query-spice');
4131
4132     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4133 }
4134
4135 sub qmp_socket {
4136     my ($vmid, $qga) = @_;
4137     my $sockettype = $qga ? 'qga' : 'qmp';
4138     return "${var_run_tmpdir}/$vmid.$sockettype";
4139 }
4140
4141 sub pidfile_name {
4142     my ($vmid) = @_;
4143     return "${var_run_tmpdir}/$vmid.pid";
4144 }
4145
4146 sub vm_devices_list {
4147     my ($vmid) = @_;
4148
4149     my $res = vm_mon_cmd($vmid, 'query-pci');
4150     my $devices_to_check = [];
4151     my $devices = {};
4152     foreach my $pcibus (@$res) {
4153         push @$devices_to_check, @{$pcibus->{devices}},
4154     }
4155
4156     while (@$devices_to_check) {
4157         my $to_check = [];
4158         for my $d (@$devices_to_check) {
4159             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4160             next if !$d->{'pci_bridge'};
4161
4162             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4163             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4164         }
4165         $devices_to_check = $to_check;
4166     }
4167
4168     my $resblock = vm_mon_cmd($vmid, 'query-block');
4169     foreach my $block (@$resblock) {
4170         if($block->{device} =~ m/^drive-(\S+)/){
4171                 $devices->{$1} = 1;
4172         }
4173     }
4174
4175     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4176     foreach my $mice (@$resmice) {
4177         if ($mice->{name} eq 'QEMU HID Tablet') {
4178             $devices->{tablet} = 1;
4179             last;
4180         }
4181     }
4182
4183     # for usb devices there is no query-usb
4184     # but we can iterate over the entries in
4185     # qom-list path=/machine/peripheral
4186     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4187     foreach my $per (@$resperipheral) {
4188         if ($per->{name} =~ m/^usb\d+$/) {
4189             $devices->{$per->{name}} = 1;
4190         }
4191     }
4192
4193     return $devices;
4194 }
4195
4196 sub vm_deviceplug {
4197     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4198
4199     my $q35 = machine_type_is_q35($conf);
4200
4201     my $devices_list = vm_devices_list($vmid);
4202     return 1 if defined($devices_list->{$deviceid});
4203
4204     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4205
4206     if ($deviceid eq 'tablet') {
4207
4208         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4209
4210     } elsif ($deviceid eq 'keyboard') {
4211
4212         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4213
4214     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4215
4216         die "usb hotplug currently not reliable\n";
4217         # since we can't reliably hot unplug all added usb devices
4218         # and usb passthrough disables live migration
4219         # we disable usb hotplugging for now
4220         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4221
4222     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4223
4224         qemu_iothread_add($vmid, $deviceid, $device);
4225
4226         qemu_driveadd($storecfg, $vmid, $device);
4227         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4228
4229         qemu_deviceadd($vmid, $devicefull);
4230         eval { qemu_deviceaddverify($vmid, $deviceid); };
4231         if (my $err = $@) {
4232             eval { qemu_drivedel($vmid, $deviceid); };
4233             warn $@ if $@;
4234             die $err;
4235         }
4236
4237     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4238
4239
4240         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4241         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4242         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4243
4244         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4245
4246         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4247             qemu_iothread_add($vmid, $deviceid, $device);
4248             $devicefull .= ",iothread=iothread-$deviceid";
4249         }
4250
4251         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4252             $devicefull .= ",num_queues=$device->{queues}";
4253         }
4254
4255         qemu_deviceadd($vmid, $devicefull);
4256         qemu_deviceaddverify($vmid, $deviceid);
4257
4258     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4259
4260         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4261         qemu_driveadd($storecfg, $vmid, $device);
4262
4263         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4264         eval { qemu_deviceadd($vmid, $devicefull); };
4265         if (my $err = $@) {
4266             eval { qemu_drivedel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270
4271     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4272
4273         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4274
4275         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4276         my $use_old_bios_files = undef;
4277         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4278
4279         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4280         qemu_deviceadd($vmid, $netdevicefull);
4281         eval {
4282             qemu_deviceaddverify($vmid, $deviceid);
4283             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4284         };
4285         if (my $err = $@) {
4286             eval { qemu_netdevdel($vmid, $deviceid); };
4287             warn $@ if $@;
4288             die $err;
4289         }
4290
4291     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4292
4293         my $bridgeid = $2;
4294         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4295         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4296
4297         qemu_deviceadd($vmid, $devicefull);
4298         qemu_deviceaddverify($vmid, $deviceid);
4299
4300     } else {
4301         die "can't hotplug device '$deviceid'\n";
4302     }
4303
4304     return 1;
4305 }
4306
4307 # fixme: this should raise exceptions on error!
4308 sub vm_deviceunplug {
4309     my ($vmid, $conf, $deviceid) = @_;
4310
4311     my $devices_list = vm_devices_list($vmid);
4312     return 1 if !defined($devices_list->{$deviceid});
4313
4314     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4315
4316     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4317
4318         qemu_devicedel($vmid, $deviceid);
4319
4320     } elsif ($deviceid =~ m/^usb\d+$/) {
4321
4322         die "usb hotplug currently not reliable\n";
4323         # when unplugging usb devices this way,
4324         # there may be remaining usb controllers/hubs
4325         # so we disable it for now
4326         qemu_devicedel($vmid, $deviceid);
4327         qemu_devicedelverify($vmid, $deviceid);
4328
4329     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4330
4331         qemu_devicedel($vmid, $deviceid);
4332         qemu_devicedelverify($vmid, $deviceid);
4333         qemu_drivedel($vmid, $deviceid);
4334         qemu_iothread_del($conf, $vmid, $deviceid);
4335
4336     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4337
4338         qemu_devicedel($vmid, $deviceid);
4339         qemu_devicedelverify($vmid, $deviceid);
4340         qemu_iothread_del($conf, $vmid, $deviceid);
4341
4342     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4343
4344         qemu_devicedel($vmid, $deviceid);
4345         qemu_drivedel($vmid, $deviceid);
4346         qemu_deletescsihw($conf, $vmid, $deviceid);
4347
4348     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4349
4350         qemu_devicedel($vmid, $deviceid);
4351         qemu_devicedelverify($vmid, $deviceid);
4352         qemu_netdevdel($vmid, $deviceid);
4353
4354     } else {
4355         die "can't unplug device '$deviceid'\n";
4356     }
4357
4358     return 1;
4359 }
4360
4361 sub qemu_deviceadd {
4362     my ($vmid, $devicefull) = @_;
4363
4364     $devicefull = "driver=".$devicefull;
4365     my %options =  split(/[=,]/, $devicefull);
4366
4367     vm_mon_cmd($vmid, "device_add" , %options);
4368 }
4369
4370 sub qemu_devicedel {
4371     my ($vmid, $deviceid) = @_;
4372
4373     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4374 }
4375
4376 sub qemu_iothread_add {
4377     my($vmid, $deviceid, $device) = @_;
4378
4379     if ($device->{iothread}) {
4380         my $iothreads = vm_iothreads_list($vmid);
4381         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4382     }
4383 }
4384
4385 sub qemu_iothread_del {
4386     my($conf, $vmid, $deviceid) = @_;
4387
4388     my $confid = $deviceid;
4389     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4390         $confid = 'scsi' . $1;
4391     }
4392     my $device = parse_drive($confid, $conf->{$confid});
4393     if ($device->{iothread}) {
4394         my $iothreads = vm_iothreads_list($vmid);
4395         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4396     }
4397 }
4398
4399 sub qemu_objectadd {
4400     my($vmid, $objectid, $qomtype) = @_;
4401
4402     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4403
4404     return 1;
4405 }
4406
4407 sub qemu_objectdel {
4408     my($vmid, $objectid) = @_;
4409
4410     vm_mon_cmd($vmid, "object-del", id => $objectid);
4411
4412     return 1;
4413 }
4414
4415 sub qemu_driveadd {
4416     my ($storecfg, $vmid, $device) = @_;
4417
4418     my $drive = print_drive_full($storecfg, $vmid, $device);
4419     $drive =~ s/\\/\\\\/g;
4420     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4421
4422     # If the command succeeds qemu prints: "OK"
4423     return 1 if $ret =~ m/OK/s;
4424
4425     die "adding drive failed: $ret\n";
4426 }
4427
4428 sub qemu_drivedel {
4429     my($vmid, $deviceid) = @_;
4430
4431     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4432     $ret =~ s/^\s+//;
4433
4434     return 1 if $ret eq "";
4435
4436     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4437     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4438
4439     die "deleting drive $deviceid failed : $ret\n";
4440 }
4441
4442 sub qemu_deviceaddverify {
4443     my ($vmid, $deviceid) = @_;
4444
4445     for (my $i = 0; $i <= 5; $i++) {
4446          my $devices_list = vm_devices_list($vmid);
4447          return 1 if defined($devices_list->{$deviceid});
4448          sleep 1;
4449     }
4450
4451     die "error on hotplug device '$deviceid'\n";
4452 }
4453
4454
4455 sub qemu_devicedelverify {
4456     my ($vmid, $deviceid) = @_;
4457
4458     # need to verify that the device is correctly removed as device_del
4459     # is async and empty return is not reliable
4460
4461     for (my $i = 0; $i <= 5; $i++) {
4462          my $devices_list = vm_devices_list($vmid);
4463          return 1 if !defined($devices_list->{$deviceid});
4464          sleep 1;
4465     }
4466
4467     die "error on hot-unplugging device '$deviceid'\n";
4468 }
4469
4470 sub qemu_findorcreatescsihw {
4471     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4472
4473     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4474
4475     my $scsihwid="$controller_prefix$controller";
4476     my $devices_list = vm_devices_list($vmid);
4477
4478     if(!defined($devices_list->{$scsihwid})) {
4479         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4480     }
4481
4482     return 1;
4483 }
4484
4485 sub qemu_deletescsihw {
4486     my ($conf, $vmid, $opt) = @_;
4487
4488     my $device = parse_drive($opt, $conf->{$opt});
4489
4490     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4491         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4492         return 1;
4493     }
4494
4495     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4496
4497     my $devices_list = vm_devices_list($vmid);
4498     foreach my $opt (keys %{$devices_list}) {
4499         if (PVE::QemuServer::is_valid_drivename($opt)) {
4500             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4501             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4502                 return 1;
4503             }
4504         }
4505     }
4506
4507     my $scsihwid="scsihw$controller";
4508
4509     vm_deviceunplug($vmid, $conf, $scsihwid);
4510
4511     return 1;
4512 }
4513
4514 sub qemu_add_pci_bridge {
4515     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4516
4517     my $bridges = {};
4518
4519     my $bridgeid;
4520
4521     print_pci_addr($device, $bridges, $arch, $machine_type);
4522
4523     while (my ($k, $v) = each %$bridges) {
4524         $bridgeid = $k;
4525     }
4526     return 1 if !defined($bridgeid) || $bridgeid < 1;
4527
4528     my $bridge = "pci.$bridgeid";
4529     my $devices_list = vm_devices_list($vmid);
4530
4531     if (!defined($devices_list->{$bridge})) {
4532         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4533     }
4534
4535     return 1;
4536 }
4537
4538 sub qemu_set_link_status {
4539     my ($vmid, $device, $up) = @_;
4540
4541     vm_mon_cmd($vmid, "set_link", name => $device,
4542                up => $up ? JSON::true : JSON::false);
4543 }
4544
4545 sub qemu_netdevadd {
4546     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4547
4548     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4549     my %options =  split(/[=,]/, $netdev);
4550
4551     vm_mon_cmd($vmid, "netdev_add",  %options);
4552     return 1;
4553 }
4554
4555 sub qemu_netdevdel {
4556     my ($vmid, $deviceid) = @_;
4557
4558     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4559 }
4560
4561 sub qemu_usb_hotplug {
4562     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4563
4564     return if !$device;
4565
4566     # remove the old one first
4567     vm_deviceunplug($vmid, $conf, $deviceid);
4568
4569     # check if xhci controller is necessary and available
4570     if ($device->{usb3}) {
4571
4572         my $devicelist = vm_devices_list($vmid);
4573
4574         if (!$devicelist->{xhci}) {
4575             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4576             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4577         }
4578     }
4579     my $d = parse_usb_device($device->{host});
4580     $d->{usb3} = $device->{usb3};
4581
4582     # add the new one
4583     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4584 }
4585
4586 sub qemu_cpu_hotplug {
4587     my ($vmid, $conf, $vcpus) = @_;
4588
4589     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4590
4591     my $sockets = 1;
4592     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4593     $sockets = $conf->{sockets} if  $conf->{sockets};
4594     my $cores = $conf->{cores} || 1;
4595     my $maxcpus = $sockets * $cores;
4596
4597     $vcpus = $maxcpus if !$vcpus;
4598
4599     die "you can't add more vcpus than maxcpus\n"
4600         if $vcpus > $maxcpus;
4601
4602     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4603
4604     if ($vcpus < $currentvcpus) {
4605
4606         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4607
4608             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4609                 qemu_devicedel($vmid, "cpu$i");
4610                 my $retry = 0;
4611                 my $currentrunningvcpus = undef;
4612                 while (1) {
4613                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4614                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4615                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4616                     $retry++;
4617                     sleep 1;
4618                 }
4619                 #update conf after each succesfull cpu unplug
4620                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4621                 PVE::QemuConfig->write_config($vmid, $conf);
4622             }
4623         } else {
4624             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4625         }
4626
4627         return;
4628     }
4629
4630     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4631     die "vcpus in running vm does not match its configuration\n"
4632         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4633
4634     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4635
4636         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4637             my $cpustr = print_cpu_device($conf, $i);
4638             qemu_deviceadd($vmid, $cpustr);
4639
4640             my $retry = 0;
4641             my $currentrunningvcpus = undef;
4642             while (1) {
4643                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4644                 last if scalar(@{$currentrunningvcpus}) == $i;
4645                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4646                 sleep 1;
4647                 $retry++;
4648             }
4649             #update conf after each succesfull cpu hotplug
4650             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4651             PVE::QemuConfig->write_config($vmid, $conf);
4652         }
4653     } else {
4654
4655         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4656             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4657         }
4658     }
4659 }
4660
4661 sub qemu_block_set_io_throttle {
4662     my ($vmid, $deviceid,
4663         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4664         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4665         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4666         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4667
4668     return if !check_running($vmid) ;
4669
4670     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4671         bps => int($bps),
4672         bps_rd => int($bps_rd),
4673         bps_wr => int($bps_wr),
4674         iops => int($iops),
4675         iops_rd => int($iops_rd),
4676         iops_wr => int($iops_wr),
4677         bps_max => int($bps_max),
4678         bps_rd_max => int($bps_rd_max),
4679         bps_wr_max => int($bps_wr_max),
4680         iops_max => int($iops_max),
4681         iops_rd_max => int($iops_rd_max),
4682         iops_wr_max => int($iops_wr_max),
4683         bps_max_length => int($bps_max_length),
4684         bps_rd_max_length => int($bps_rd_max_length),
4685         bps_wr_max_length => int($bps_wr_max_length),
4686         iops_max_length => int($iops_max_length),
4687         iops_rd_max_length => int($iops_rd_max_length),
4688         iops_wr_max_length => int($iops_wr_max_length),
4689     );
4690
4691 }
4692
4693 # old code, only used to shutdown old VM after update
4694 sub __read_avail {
4695     my ($fh, $timeout) = @_;
4696
4697     my $sel = new IO::Select;
4698     $sel->add($fh);
4699
4700     my $res = '';
4701     my $buf;
4702
4703     my @ready;
4704     while (scalar (@ready = $sel->can_read($timeout))) {
4705         my $count;
4706         if ($count = $fh->sysread($buf, 8192)) {
4707             if ($buf =~ /^(.*)\(qemu\) $/s) {
4708                 $res .= $1;
4709                 last;
4710             } else {
4711                 $res .= $buf;
4712             }
4713         } else {
4714             if (!defined($count)) {
4715                 die "$!\n";
4716             }
4717             last;
4718         }
4719     }
4720
4721     die "monitor read timeout\n" if !scalar(@ready);
4722
4723     return $res;
4724 }
4725
4726 sub qemu_block_resize {
4727     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4728
4729     my $running = check_running($vmid);
4730
4731     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4732
4733     return if !$running;
4734
4735     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4736
4737 }
4738
4739 sub qemu_volume_snapshot {
4740     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4741
4742     my $running = check_running($vmid);
4743
4744     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4745         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4746     } else {
4747         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4748     }
4749 }
4750
4751 sub qemu_volume_snapshot_delete {
4752     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4753
4754     my $running = check_running($vmid);
4755
4756     if($running) {
4757
4758         $running = undef;
4759         my $conf = PVE::QemuConfig->load_config($vmid);
4760         foreach_drive($conf, sub {
4761             my ($ds, $drive) = @_;
4762             $running = 1 if $drive->{file} eq $volid;
4763         });
4764     }
4765
4766     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4767         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4768     } else {
4769         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4770     }
4771 }
4772
4773 sub set_migration_caps {
4774     my ($vmid) = @_;
4775
4776     my $cap_ref = [];
4777
4778     my $enabled_cap = {
4779         "auto-converge" => 1,
4780         "xbzrle" => 1,
4781         "x-rdma-pin-all" => 0,
4782         "zero-blocks" => 0,
4783         "compress" => 0
4784     };
4785
4786     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4787
4788     for my $supported_capability (@$supported_capabilities) {
4789         push @$cap_ref, {
4790             capability => $supported_capability->{capability},
4791             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4792         };
4793     }
4794
4795     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4796 }
4797
4798 my $fast_plug_option = {
4799     'lock' => 1,
4800     'name' => 1,
4801     'onboot' => 1,
4802     'shares' => 1,
4803     'startup' => 1,
4804     'description' => 1,
4805     'protection' => 1,
4806     'vmstatestorage' => 1,
4807     'hookscript' => 1,
4808 };
4809
4810 # hotplug changes in [PENDING]
4811 # $selection hash can be used to only apply specified options, for
4812 # example: { cores => 1 } (only apply changed 'cores')
4813 # $errors ref is used to return error messages
4814 sub vmconfig_hotplug_pending {
4815     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4816
4817     my $defaults = load_defaults();
4818     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4819
4820     # commit values which do not have any impact on running VM first
4821     # Note: those option cannot raise errors, we we do not care about
4822     # $selection and always apply them.
4823
4824     my $add_error = sub {
4825         my ($opt, $msg) = @_;
4826         $errors->{$opt} = "hotplug problem - $msg";
4827     };
4828
4829     my $changes = 0;
4830     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4831         if ($fast_plug_option->{$opt}) {
4832             $conf->{$opt} = $conf->{pending}->{$opt};
4833             delete $conf->{pending}->{$opt};
4834             $changes = 1;
4835         }
4836     }
4837
4838     if ($changes) {
4839         PVE::QemuConfig->write_config($vmid, $conf);
4840         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4841     }
4842
4843     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4844
4845     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4846     foreach my $opt (sort keys %$pending_delete_hash) {
4847         next if $selection && !$selection->{$opt};
4848         my $force = $pending_delete_hash->{$opt}->{force};
4849         eval {
4850             if ($opt eq 'hotplug') {
4851                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4852             } elsif ($opt eq 'tablet') {
4853                 die "skip\n" if !$hotplug_features->{usb};
4854                 if ($defaults->{tablet}) {
4855                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4856                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4857                         if $arch eq 'aarch64';
4858                 } else {
4859                     vm_deviceunplug($vmid, $conf, 'tablet');
4860                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4861                 }
4862             } elsif ($opt =~ m/^usb\d+/) {
4863                 die "skip\n";
4864                 # since we cannot reliably hot unplug usb devices
4865                 # we are disabling it
4866                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4867                 vm_deviceunplug($vmid, $conf, $opt);
4868             } elsif ($opt eq 'vcpus') {
4869                 die "skip\n" if !$hotplug_features->{cpu};
4870                 qemu_cpu_hotplug($vmid, $conf, undef);
4871             } elsif ($opt eq 'balloon') {
4872                 # enable balloon device is not hotpluggable
4873                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4874                 # here we reset the ballooning value to memory
4875                 my $balloon = $conf->{memory} || $defaults->{memory};
4876                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4877             } elsif ($fast_plug_option->{$opt}) {
4878                 # do nothing
4879             } elsif ($opt =~ m/^net(\d+)$/) {
4880                 die "skip\n" if !$hotplug_features->{network};
4881                 vm_deviceunplug($vmid, $conf, $opt);
4882             } elsif (is_valid_drivename($opt)) {
4883                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4884                 vm_deviceunplug($vmid, $conf, $opt);
4885                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4886             } elsif ($opt =~ m/^memory$/) {
4887                 die "skip\n" if !$hotplug_features->{memory};
4888                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4889             } elsif ($opt eq 'cpuunits') {
4890                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4891             } elsif ($opt eq 'cpulimit') {
4892                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4893             } else {
4894                 die "skip\n";
4895             }
4896         };
4897         if (my $err = $@) {
4898             &$add_error($opt, $err) if $err ne "skip\n";
4899         } else {
4900             # save new config if hotplug was successful
4901             delete $conf->{$opt};
4902             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4903             PVE::QemuConfig->write_config($vmid, $conf);
4904             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4905         }
4906     }
4907
4908     my $apply_pending_cloudinit;
4909     $apply_pending_cloudinit = sub {
4910         my ($key, $value) = @_;
4911         $apply_pending_cloudinit = sub {}; # once is enough
4912
4913         my @cloudinit_opts = keys %$confdesc_cloudinit;
4914         foreach my $opt (keys %{$conf->{pending}}) {
4915             next if !grep { $_ eq $opt } @cloudinit_opts;
4916             $conf->{$opt} = delete $conf->{pending}->{$opt};
4917         }
4918
4919         my $new_conf = { %$conf };
4920         $new_conf->{$key} = $value;
4921         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4922     };
4923
4924     foreach my $opt (keys %{$conf->{pending}}) {
4925         next if $selection && !$selection->{$opt};
4926         my $value = $conf->{pending}->{$opt};
4927         eval {
4928             if ($opt eq 'hotplug') {
4929                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4930             } elsif ($opt eq 'tablet') {
4931                 die "skip\n" if !$hotplug_features->{usb};
4932                 if ($value == 1) {
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4934                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4935                         if $arch eq 'aarch64';
4936                 } elsif ($value == 0) {
4937                     vm_deviceunplug($vmid, $conf, 'tablet');
4938                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4939                 }
4940             } elsif ($opt =~ m/^usb\d+$/) {
4941                 die "skip\n";
4942                 # since we cannot reliably hot unplug usb devices
4943                 # we are disabling it
4944                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4945                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4946                 die "skip\n" if !$d;
4947                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4948             } elsif ($opt eq 'vcpus') {
4949                 die "skip\n" if !$hotplug_features->{cpu};
4950                 qemu_cpu_hotplug($vmid, $conf, $value);
4951             } elsif ($opt eq 'balloon') {
4952                 # enable/disable balloning device is not hotpluggable
4953                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4954                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4955                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4956
4957                 # allow manual ballooning if shares is set to zero
4958                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4959                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4960                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4961                 }
4962             } elsif ($opt =~ m/^net(\d+)$/) {
4963                 # some changes can be done without hotplug
4964                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4965                                     $vmid, $opt, $value, $arch, $machine_type);
4966             } elsif (is_valid_drivename($opt)) {
4967                 # some changes can be done without hotplug
4968                 my $drive = parse_drive($opt, $value);
4969                 if (drive_is_cloudinit($drive)) {
4970                     &$apply_pending_cloudinit($opt, $value);
4971                 }
4972                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4973                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4974             } elsif ($opt =~ m/^memory$/) { #dimms
4975                 die "skip\n" if !$hotplug_features->{memory};
4976                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4977             } elsif ($opt eq 'cpuunits') {
4978                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4979             } elsif ($opt eq 'cpulimit') {
4980                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4981                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4982             } else {
4983                 die "skip\n";  # skip non-hot-pluggable options
4984             }
4985         };
4986         if (my $err = $@) {
4987             &$add_error($opt, $err) if $err ne "skip\n";
4988         } else {
4989             # save new config if hotplug was successful
4990             $conf->{$opt} = $value;
4991             delete $conf->{pending}->{$opt};
4992             PVE::QemuConfig->write_config($vmid, $conf);
4993             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4994         }
4995     }
4996 }
4997
4998 sub try_deallocate_drive {
4999     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5000
5001     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5002         my $volid = $drive->{file};
5003         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5004             my $sid = PVE::Storage::parse_volume_id($volid);
5005             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5006
5007             # check if the disk is really unused
5008             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5009                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5010             PVE::Storage::vdisk_free($storecfg, $volid);
5011             return 1;
5012         } else {
5013             # If vm is not owner of this disk remove from config
5014             return 1;
5015         }
5016     }
5017
5018     return undef;
5019 }
5020
5021 sub vmconfig_delete_or_detach_drive {
5022     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5023
5024     my $drive = parse_drive($opt, $conf->{$opt});
5025
5026     my $rpcenv = PVE::RPCEnvironment::get();
5027     my $authuser = $rpcenv->get_user();
5028
5029     if ($force) {
5030         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5031         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5032     } else {
5033         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5034     }
5035 }
5036
5037
5038
5039 sub vmconfig_apply_pending {
5040     my ($vmid, $conf, $storecfg) = @_;
5041
5042     # cold plug
5043
5044     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5045     foreach my $opt (sort keys %$pending_delete_hash) {
5046         die "internal error" if $opt =~ m/^unused/;
5047         my $force = $pending_delete_hash->{$opt}->{force};
5048         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5049         if (!defined($conf->{$opt})) {
5050             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5051             PVE::QemuConfig->write_config($vmid, $conf);
5052         } elsif (is_valid_drivename($opt)) {
5053             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5054             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5055             delete $conf->{$opt};
5056             PVE::QemuConfig->write_config($vmid, $conf);
5057         } else {
5058             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5059             delete $conf->{$opt};
5060             PVE::QemuConfig->write_config($vmid, $conf);
5061         }
5062     }
5063
5064     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5065
5066     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5067         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5068
5069         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5070             # skip if nothing changed
5071         } elsif (is_valid_drivename($opt)) {
5072             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5073                 if defined($conf->{$opt});
5074             $conf->{$opt} = $conf->{pending}->{$opt};
5075         } else {
5076             $conf->{$opt} = $conf->{pending}->{$opt};
5077         }
5078
5079         delete $conf->{pending}->{$opt};
5080         PVE::QemuConfig->write_config($vmid, $conf);
5081     }
5082 }
5083
5084 my $safe_num_ne = sub {
5085     my ($a, $b) = @_;
5086
5087     return 0 if !defined($a) && !defined($b);
5088     return 1 if !defined($a);
5089     return 1 if !defined($b);
5090
5091     return $a != $b;
5092 };
5093
5094 my $safe_string_ne = sub {
5095     my ($a, $b) = @_;
5096
5097     return 0 if !defined($a) && !defined($b);
5098     return 1 if !defined($a);
5099     return 1 if !defined($b);
5100
5101     return $a ne $b;
5102 };
5103
5104 sub vmconfig_update_net {
5105     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5106
5107     my $newnet = parse_net($value);
5108
5109     if ($conf->{$opt}) {
5110         my $oldnet = parse_net($conf->{$opt});
5111
5112         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5113             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5114             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5115             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5116
5117             # for non online change, we try to hot-unplug
5118             die "skip\n" if !$hotplug;
5119             vm_deviceunplug($vmid, $conf, $opt);
5120         } else {
5121
5122             die "internal error" if $opt !~ m/net(\d+)/;
5123             my $iface = "tap${vmid}i$1";
5124
5125             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5126                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5127                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5128                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5129                 PVE::Network::tap_unplug($iface);
5130                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5131             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5132                 # Rate can be applied on its own but any change above needs to
5133                 # include the rate in tap_plug since OVS resets everything.
5134                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5135             }
5136
5137             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5138                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5139             }
5140
5141             return 1;
5142         }
5143     }
5144
5145     if ($hotplug) {
5146         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5147     } else {
5148         die "skip\n";
5149     }
5150 }
5151
5152 sub vmconfig_update_disk {
5153     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5154
5155     # fixme: do we need force?
5156
5157     my $drive = parse_drive($opt, $value);
5158
5159     if ($conf->{$opt}) {
5160
5161         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5162
5163             my $media = $drive->{media} || 'disk';
5164             my $oldmedia = $old_drive->{media} || 'disk';
5165             die "unable to change media type\n" if $media ne $oldmedia;
5166
5167             if (!drive_is_cdrom($old_drive)) {
5168
5169                 if ($drive->{file} ne $old_drive->{file}) {
5170
5171                     die "skip\n" if !$hotplug;
5172
5173                     # unplug and register as unused
5174                     vm_deviceunplug($vmid, $conf, $opt);
5175                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5176
5177                 } else {
5178                     # update existing disk
5179
5180                     # skip non hotpluggable value
5181                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5182                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5183                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5184                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5185                         die "skip\n";
5186                     }
5187
5188                     # apply throttle
5189                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5190                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5191                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5192                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5193                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5194                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5195                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5196                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5197                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5198                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5199                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5200                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5201                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5202                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5203                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5204                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5205                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5206                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5207
5208                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5209                                                    ($drive->{mbps} || 0)*1024*1024,
5210                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5211                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5212                                                    $drive->{iops} || 0,
5213                                                    $drive->{iops_rd} || 0,
5214                                                    $drive->{iops_wr} || 0,
5215                                                    ($drive->{mbps_max} || 0)*1024*1024,
5216                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5217                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5218                                                    $drive->{iops_max} || 0,
5219                                                    $drive->{iops_rd_max} || 0,
5220                                                    $drive->{iops_wr_max} || 0,
5221                                                    $drive->{bps_max_length} || 1,
5222                                                    $drive->{bps_rd_max_length} || 1,
5223                                                    $drive->{bps_wr_max_length} || 1,
5224                                                    $drive->{iops_max_length} || 1,
5225                                                    $drive->{iops_rd_max_length} || 1,
5226                                                    $drive->{iops_wr_max_length} || 1);
5227
5228                     }
5229
5230                     return 1;
5231                 }
5232
5233             } else { # cdrom
5234
5235                 if ($drive->{file} eq 'none') {
5236                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5237                     if (drive_is_cloudinit($old_drive)) {
5238                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5239                     }
5240                 } else {
5241                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5242                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5243                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5244                 }
5245
5246                 return 1;
5247             }
5248         }
5249     }
5250
5251     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5252     # hotplug new disks
5253     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5254     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5255 }
5256
5257 sub vm_start {
5258     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5259         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5260
5261     PVE::QemuConfig->lock_config($vmid, sub {
5262         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5263
5264         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5265
5266         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5267
5268         PVE::QemuConfig->check_lock($conf)
5269             if !($skiplock || $is_suspended);
5270
5271         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5272
5273         # clean up leftover reboot request files
5274         eval { clear_reboot_request($vmid); };
5275         warn $@ if $@;
5276
5277         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5278             vmconfig_apply_pending($vmid, $conf, $storecfg);
5279             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5280         }
5281
5282         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5283
5284         my $defaults = load_defaults();
5285
5286         # set environment variable useful inside network script
5287         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5288
5289         my $local_volumes = {};
5290
5291         if ($targetstorage) {
5292             foreach_drive($conf, sub {
5293                 my ($ds, $drive) = @_;
5294
5295                 return if drive_is_cdrom($drive);
5296
5297                 my $volid = $drive->{file};
5298
5299                 return if !$volid;
5300
5301                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5302
5303                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5304                 return if $scfg->{shared};
5305                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5306             });
5307
5308             my $format = undef;
5309
5310             foreach my $opt (sort keys %$local_volumes) {
5311
5312                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5313                 my $drive = parse_drive($opt, $conf->{$opt});
5314
5315                 #if remote storage is specified, use default format
5316                 if ($targetstorage && $targetstorage ne "1") {
5317                     $storeid = $targetstorage;
5318                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5319                     $format = $defFormat;
5320                 } else {
5321                     #else we use same format than original
5322                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5323                     $format = qemu_img_format($scfg, $volid);
5324                 }
5325
5326                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5327                 my $newdrive = $drive;
5328                 $newdrive->{format} = $format;
5329                 $newdrive->{file} = $newvolid;
5330                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5331                 $local_volumes->{$opt} = $drivestr;
5332                 #pass drive to conf for command line
5333                 $conf->{$opt} = $drivestr;
5334             }
5335         }
5336
5337         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5338
5339         if ($is_suspended) {
5340             # enforce machine type on suspended vm to ensure HW compatibility
5341             $forcemachine = $conf->{runningmachine};
5342             print "Resuming suspended VM\n";
5343         }
5344
5345         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5346
5347         my $migrate_uri;
5348         if ($statefile) {
5349             if ($statefile eq 'tcp') {
5350                 my $localip = "localhost";
5351                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5352                 my $nodename = PVE::INotify::nodename();
5353
5354                 if (!defined($migration_type)) {
5355                     if (defined($datacenterconf->{migration}->{type})) {
5356                         $migration_type = $datacenterconf->{migration}->{type};
5357                     } else {
5358                         $migration_type = 'secure';
5359                     }
5360                 }
5361
5362                 if ($migration_type eq 'insecure') {
5363                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5364                     if ($migrate_network_addr) {
5365                         $localip = $migrate_network_addr;
5366                     } else {
5367                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5368                     }
5369
5370                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5371                 }
5372
5373                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5374                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5375                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5376                 push @$cmd, '-incoming', $migrate_uri;
5377                 push @$cmd, '-S';
5378
5379             } elsif ($statefile eq 'unix') {
5380                 # should be default for secure migrations as a ssh TCP forward
5381                 # tunnel is not deterministic reliable ready and fails regurarly
5382                 # to set up in time, so use UNIX socket forwards
5383                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5384                 unlink $socket_addr;
5385
5386                 $migrate_uri = "unix:$socket_addr";
5387
5388                 push @$cmd, '-incoming', $migrate_uri;
5389                 push @$cmd, '-S';
5390
5391             } elsif (-e $statefile) {
5392                 push @$cmd, '-loadstate', $statefile;
5393             } else {
5394                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5395                 push @$vollist, $statefile;
5396                 push @$cmd, '-loadstate', $statepath;
5397             }
5398         } elsif ($paused) {
5399             push @$cmd, '-S';
5400         }
5401
5402         # host pci devices
5403         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5404           my $d = parse_hostpci($conf->{"hostpci$i"});
5405           next if !$d;
5406           my $pcidevices = $d->{pciid};
5407           foreach my $pcidevice (@$pcidevices) {
5408                 my $pciid = $pcidevice->{id};
5409
5410                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5411                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5412                 die "no pci device info for device '$pciid'\n" if !$info;
5413
5414                 if ($d->{mdev}) {
5415                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5416                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5417                 } else {
5418                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5419                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5420                     die "can't reset pci device '$pciid'\n"
5421                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5422                 }
5423           }
5424         }
5425
5426         PVE::Storage::activate_volumes($storecfg, $vollist);
5427
5428         eval {
5429             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5430                 outfunc => sub {}, errfunc => sub {});
5431         };
5432         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5433         # timeout should be more than enough here...
5434         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5435
5436         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5437                                                   : $defaults->{cpuunits};
5438
5439         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5440         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5441
5442         my %properties = (
5443             Slice => 'qemu.slice',
5444             KillMode => 'none',
5445             CPUShares => $cpuunits
5446         );
5447
5448         if (my $cpulimit = $conf->{cpulimit}) {
5449             $properties{CPUQuota} = int($cpulimit * 100);
5450         }
5451         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5452
5453         my $run_qemu = sub {
5454             PVE::Tools::run_fork sub {
5455                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5456                 run_command($cmd, %run_params);
5457             };
5458         };
5459
5460         if ($conf->{hugepages}) {
5461
5462             my $code = sub {
5463                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5464                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5465
5466                 PVE::QemuServer::Memory::hugepages_mount();
5467                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5468
5469                 eval { $run_qemu->() };
5470                 if (my $err = $@) {
5471                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5472                     die $err;
5473                 }
5474
5475                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5476             };
5477             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5478
5479         } else {
5480             eval { $run_qemu->() };
5481         }
5482
5483         if (my $err = $@) {
5484             # deactivate volumes if start fails
5485             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5486             die "start failed: $err";
5487         }
5488
5489         print "migration listens on $migrate_uri\n" if $migrate_uri;
5490
5491         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5492             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5493             warn $@ if $@;
5494         }
5495
5496         #start nbd server for storage migration
5497         if ($targetstorage) {
5498             my $nodename = PVE::INotify::nodename();
5499             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5500             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5501             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5502             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5503
5504             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5505
5506             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5507
5508             foreach my $opt (sort keys %$local_volumes) {
5509                 my $volid = $local_volumes->{$opt};
5510                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5511                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5512                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5513             }
5514         }
5515
5516         if ($migratedfrom) {
5517             eval {
5518                 set_migration_caps($vmid);
5519             };
5520             warn $@ if $@;
5521
5522             if ($spice_port) {
5523                 print "spice listens on port $spice_port\n";
5524                 if ($spice_ticket) {
5525                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5526                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5527                 }
5528             }
5529
5530         } else {
5531             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5532                 if !$statefile && $conf->{balloon};
5533
5534             foreach my $opt (keys %$conf) {
5535                 next if $opt !~  m/^net\d+$/;
5536                 my $nicconf = parse_net($conf->{$opt});
5537                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5538             }
5539         }
5540
5541         vm_mon_cmd_nocheck($vmid, 'qom-set',
5542                     path => "machine/peripheral/balloon0",
5543                     property => "guest-stats-polling-interval",
5544                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5545
5546         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5547             print "Resumed VM, removing state\n";
5548             delete $conf->@{qw(lock vmstate runningmachine)};
5549             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5550             PVE::Storage::vdisk_free($storecfg, $vmstate);
5551             PVE::QemuConfig->write_config($vmid, $conf);
5552         }
5553
5554         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5555     });
5556 }
5557
5558 sub vm_mon_cmd {
5559     my ($vmid, $execute, %params) = @_;
5560
5561     my $cmd = { execute => $execute, arguments => \%params };
5562     vm_qmp_command($vmid, $cmd);
5563 }
5564
5565 sub vm_mon_cmd_nocheck {
5566     my ($vmid, $execute, %params) = @_;
5567
5568     my $cmd = { execute => $execute, arguments => \%params };
5569     vm_qmp_command($vmid, $cmd, 1);
5570 }
5571
5572 sub vm_qmp_command {
5573     my ($vmid, $cmd, $nocheck) = @_;
5574
5575     my $res;
5576
5577     my $timeout;
5578     if ($cmd->{arguments}) {
5579         $timeout = delete $cmd->{arguments}->{timeout};
5580     }
5581
5582     eval {
5583         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5584         my $sname = qmp_socket($vmid);
5585         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5586             my $qmpclient = PVE::QMPClient->new();
5587
5588             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5589         } else {
5590             die "unable to open monitor socket\n";
5591         }
5592     };
5593     if (my $err = $@) {
5594         syslog("err", "VM $vmid qmp command failed - $err");
5595         die $err;
5596     }
5597
5598     return $res;
5599 }
5600
5601 sub vm_human_monitor_command {
5602     my ($vmid, $cmdline) = @_;
5603
5604     my $cmd = {
5605         execute => 'human-monitor-command',
5606         arguments => { 'command-line' => $cmdline},
5607     };
5608
5609     return vm_qmp_command($vmid, $cmd);
5610 }
5611
5612 sub vm_commandline {
5613     my ($storecfg, $vmid, $snapname) = @_;
5614
5615     my $conf = PVE::QemuConfig->load_config($vmid);
5616
5617     if ($snapname) {
5618         my $snapshot = $conf->{snapshots}->{$snapname};
5619         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5620
5621         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5622
5623         $conf = $snapshot;
5624     }
5625
5626     my $defaults = load_defaults();
5627
5628     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5629
5630     return PVE::Tools::cmd2string($cmd);
5631 }
5632
5633 sub vm_reset {
5634     my ($vmid, $skiplock) = @_;
5635
5636     PVE::QemuConfig->lock_config($vmid, sub {
5637
5638         my $conf = PVE::QemuConfig->load_config($vmid);
5639
5640         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5641
5642         vm_mon_cmd($vmid, "system_reset");
5643     });
5644 }
5645
5646 sub get_vm_volumes {
5647     my ($conf) = @_;
5648
5649     my $vollist = [];
5650     foreach_volid($conf, sub {
5651         my ($volid, $attr) = @_;
5652
5653         return if $volid =~ m|^/|;
5654
5655         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5656         return if !$sid;
5657
5658         push @$vollist, $volid;
5659     });
5660
5661     return $vollist;
5662 }
5663
5664 sub vm_stop_cleanup {
5665     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5666
5667     eval {
5668
5669         if (!$keepActive) {
5670             my $vollist = get_vm_volumes($conf);
5671             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5672         }
5673
5674         foreach my $ext (qw(mon qmp pid vnc qga)) {
5675             unlink "/var/run/qemu-server/${vmid}.$ext";
5676         }
5677
5678         if ($conf->{ivshmem}) {
5679             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5680             # just delete it for now, VMs which have this already open do not
5681             # are affected, but new VMs will get a separated one. If this
5682             # becomes an issue we either add some sort of ref-counting or just
5683             # add a "don't delete on stop" flag to the ivshmem format.
5684             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5685         }
5686
5687         foreach my $key (keys %$conf) {
5688             next if $key !~ m/^hostpci(\d+)$/;
5689             my $hostpciindex = $1;
5690             my $d = parse_hostpci($conf->{$key});
5691             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5692
5693             foreach my $pci (@{$d->{pciid}}) {
5694                 my $pciid = $pci->{id};
5695                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5696             }
5697         }
5698
5699         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5700     };
5701     warn $@ if $@; # avoid errors - just warn
5702 }
5703
5704 # call only in locked context
5705 sub _do_vm_stop {
5706     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5707
5708     my $pid = check_running($vmid, $nocheck);
5709     return if !$pid;
5710
5711     my $conf;
5712     if (!$nocheck) {
5713         $conf = PVE::QemuConfig->load_config($vmid);
5714         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5715         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5716             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5717             $timeout = $opts->{down} if $opts->{down};
5718         }
5719         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5720     }
5721
5722     eval {
5723         if ($shutdown) {
5724             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5725                 vm_qmp_command($vmid, {
5726                         execute => "guest-shutdown",
5727                         arguments => { timeout => $timeout }
5728                     }, $nocheck);
5729             } else {
5730                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5731             }
5732         } else {
5733             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5734         }
5735     };
5736     my $err = $@;
5737
5738     if (!$err) {
5739         $timeout = 60 if !defined($timeout);
5740
5741         my $count = 0;
5742         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5743             $count++;
5744             sleep 1;
5745         }
5746
5747         if ($count >= $timeout) {
5748             if ($force) {
5749                 warn "VM still running - terminating now with SIGTERM\n";
5750                 kill 15, $pid;
5751             } else {
5752                 die "VM quit/powerdown failed - got timeout\n";
5753             }
5754         } else {
5755             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5756             return;
5757         }
5758     } else {
5759         if ($force) {
5760             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5761             kill 15, $pid;
5762         } else {
5763             die "VM quit/powerdown failed\n";
5764         }
5765     }
5766
5767     # wait again
5768     $timeout = 10;
5769
5770     my $count = 0;
5771     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5772         $count++;
5773         sleep 1;
5774     }
5775
5776     if ($count >= $timeout) {
5777         warn "VM still running - terminating now with SIGKILL\n";
5778         kill 9, $pid;
5779         sleep 1;
5780     }
5781
5782     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5783 }
5784
5785 # Note: use $nocheck to skip tests if VM configuration file exists.
5786 # We need that when migration VMs to other nodes (files already moved)
5787 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5788 sub vm_stop {
5789     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5790
5791     $force = 1 if !defined($force) && !$shutdown;
5792
5793     if ($migratedfrom){
5794         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5795         kill 15, $pid if $pid;
5796         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5797         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5798         return;
5799     }
5800
5801     PVE::QemuConfig->lock_config($vmid, sub {
5802         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5803    });
5804 }
5805
5806 sub vm_reboot {
5807     my ($vmid, $timeout) = @_;
5808
5809     PVE::QemuConfig->lock_config($vmid, sub {
5810
5811         # only reboot if running, as qmeventd starts it again on a stop event
5812         return if !check_running($vmid);
5813
5814         create_reboot_request($vmid);
5815
5816         my $storecfg = PVE::Storage::config();
5817         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5818
5819    });
5820 }
5821
5822 sub vm_suspend {
5823     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5824
5825     my $conf;
5826     my $path;
5827     my $storecfg;
5828     my $vmstate;
5829
5830     PVE::QemuConfig->lock_config($vmid, sub {
5831
5832         $conf = PVE::QemuConfig->load_config($vmid);
5833
5834         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5835         PVE::QemuConfig->check_lock($conf)
5836             if !($skiplock || $is_backing_up);
5837
5838         die "cannot suspend to disk during backup\n"
5839             if $is_backing_up && $includestate;
5840
5841         if ($includestate) {
5842             $conf->{lock} = 'suspending';
5843             my $date = strftime("%Y-%m-%d", localtime(time()));
5844             $storecfg = PVE::Storage::config();
5845             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5846             $path = PVE::Storage::path($storecfg, $vmstate);
5847             PVE::QemuConfig->write_config($vmid, $conf);
5848         } else {
5849             vm_mon_cmd($vmid, "stop");
5850         }
5851     });
5852
5853     if ($includestate) {
5854         # save vm state
5855         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5856
5857         eval {
5858             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5859             for(;;) {
5860                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5861                 if (!$state->{status}) {
5862                     die "savevm not active\n";
5863                 } elsif ($state->{status} eq 'active') {
5864                     sleep(1);
5865                     next;
5866                 } elsif ($state->{status} eq 'completed') {
5867                     print "State saved, quitting\n";
5868                     last;
5869                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5870                     die "query-savevm failed with error '$state->{error}'\n"
5871                 } else {
5872                     die "query-savevm returned status '$state->{status}'\n";
5873                 }
5874             }
5875         };
5876         my $err = $@;
5877
5878         PVE::QemuConfig->lock_config($vmid, sub {
5879             $conf = PVE::QemuConfig->load_config($vmid);
5880             if ($err) {
5881                 # cleanup, but leave suspending lock, to indicate something went wrong
5882                 eval {
5883                     vm_mon_cmd($vmid, "savevm-end");
5884                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5885                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5886                     delete $conf->@{qw(vmstate runningmachine)};
5887                     PVE::QemuConfig->write_config($vmid, $conf);
5888                 };
5889                 warn $@ if $@;
5890                 die $err;
5891             }
5892
5893             die "lock changed unexpectedly\n"
5894                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5895
5896             vm_qmp_command($vmid, { execute => "quit" });
5897             $conf->{lock} = 'suspended';
5898             PVE::QemuConfig->write_config($vmid, $conf);
5899         });
5900     }
5901 }
5902
5903 sub vm_resume {
5904     my ($vmid, $skiplock, $nocheck) = @_;
5905
5906     PVE::QemuConfig->lock_config($vmid, sub {
5907         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5908         my $res = $vm_mon_cmd->($vmid, 'query-status');
5909         my $resume_cmd = 'cont';
5910
5911         if ($res->{status} && $res->{status} eq 'suspended') {
5912             $resume_cmd = 'system_wakeup';
5913         }
5914
5915         if (!$nocheck) {
5916
5917             my $conf = PVE::QemuConfig->load_config($vmid);
5918
5919             PVE::QemuConfig->check_lock($conf)
5920                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5921         }
5922
5923         $vm_mon_cmd->($vmid, $resume_cmd);
5924     });
5925 }
5926
5927 sub vm_sendkey {
5928     my ($vmid, $skiplock, $key) = @_;
5929
5930     PVE::QemuConfig->lock_config($vmid, sub {
5931
5932         my $conf = PVE::QemuConfig->load_config($vmid);
5933
5934         # there is no qmp command, so we use the human monitor command
5935         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5936         die $res if $res ne '';
5937     });
5938 }
5939
5940 # vzdump restore implementaion
5941
5942 sub tar_archive_read_firstfile {
5943     my $archive = shift;
5944
5945     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5946
5947     # try to detect archive type first
5948     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5949         die "unable to open file '$archive'\n";
5950     my $firstfile = <$fh>;
5951     kill 15, $pid;
5952     close $fh;
5953
5954     die "ERROR: archive contaions no data\n" if !$firstfile;
5955     chomp $firstfile;
5956
5957     return $firstfile;
5958 }
5959
5960 sub tar_restore_cleanup {
5961     my ($storecfg, $statfile) = @_;
5962
5963     print STDERR "starting cleanup\n";
5964
5965     if (my $fd = IO::File->new($statfile, "r")) {
5966         while (defined(my $line = <$fd>)) {
5967             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5968                 my $volid = $2;
5969                 eval {
5970                     if ($volid =~ m|^/|) {
5971                         unlink $volid || die 'unlink failed\n';
5972                     } else {
5973                         PVE::Storage::vdisk_free($storecfg, $volid);
5974                     }
5975                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5976                 };
5977                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5978             } else {
5979                 print STDERR "unable to parse line in statfile - $line";
5980             }
5981         }
5982         $fd->close();
5983     }
5984 }
5985
5986 sub restore_archive {
5987     my ($archive, $vmid, $user, $opts) = @_;
5988
5989     my $format = $opts->{format};
5990     my $comp;
5991
5992     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5993         $format = 'tar' if !$format;
5994         $comp = 'gzip';
5995     } elsif ($archive =~ m/\.tar$/) {
5996         $format = 'tar' if !$format;
5997     } elsif ($archive =~ m/.tar.lzo$/) {
5998         $format = 'tar' if !$format;
5999         $comp = 'lzop';
6000     } elsif ($archive =~ m/\.vma$/) {
6001         $format = 'vma' if !$format;
6002     } elsif ($archive =~ m/\.vma\.gz$/) {
6003         $format = 'vma' if !$format;
6004         $comp = 'gzip';
6005     } elsif ($archive =~ m/\.vma\.lzo$/) {
6006         $format = 'vma' if !$format;
6007         $comp = 'lzop';
6008     } else {
6009         $format = 'vma' if !$format; # default
6010     }
6011
6012     # try to detect archive format
6013     if ($format eq 'tar') {
6014         return restore_tar_archive($archive, $vmid, $user, $opts);
6015     } else {
6016         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6017     }
6018 }
6019
6020 sub restore_update_config_line {
6021     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6022
6023     return if $line =~ m/^\#qmdump\#/;
6024     return if $line =~ m/^\#vzdump\#/;
6025     return if $line =~ m/^lock:/;
6026     return if $line =~ m/^unused\d+:/;
6027     return if $line =~ m/^parent:/;
6028
6029     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6030     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6031         # try to convert old 1.X settings
6032         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6033         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6034             my ($model, $macaddr) = split(/\=/, $devconfig);
6035             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6036             my $net = {
6037                 model => $model,
6038                 bridge => "vmbr$ind",
6039                 macaddr => $macaddr,
6040             };
6041             my $netstr = print_net($net);
6042
6043             print $outfd "net$cookie->{netcount}: $netstr\n";
6044             $cookie->{netcount}++;
6045         }
6046     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6047         my ($id, $netstr) = ($1, $2);
6048         my $net = parse_net($netstr);
6049         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6050         $netstr = print_net($net);
6051         print $outfd "$id: $netstr\n";
6052     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6053         my $virtdev = $1;
6054         my $value = $3;
6055         my $di = parse_drive($virtdev, $value);
6056         if (defined($di->{backup}) && !$di->{backup}) {
6057             print $outfd "#$line";
6058         } elsif ($map->{$virtdev}) {
6059             delete $di->{format}; # format can change on restore
6060             $di->{file} = $map->{$virtdev};
6061             $value = print_drive($vmid, $di);
6062             print $outfd "$virtdev: $value\n";
6063         } else {
6064             print $outfd $line;
6065         }
6066     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6067         my $vmgenid = $1;
6068         if ($vmgenid ne '0') {
6069             # always generate a new vmgenid if there was a valid one setup
6070             $vmgenid = generate_uuid();
6071         }
6072         print $outfd "vmgenid: $vmgenid\n";
6073     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6074         my ($uuid, $uuid_str);
6075         UUID::generate($uuid);
6076         UUID::unparse($uuid, $uuid_str);
6077         my $smbios1 = parse_smbios1($2);
6078         $smbios1->{uuid} = $uuid_str;
6079         print $outfd $1.print_smbios1($smbios1)."\n";
6080     } else {
6081         print $outfd $line;
6082     }
6083 }
6084
6085 sub scan_volids {
6086     my ($cfg, $vmid) = @_;
6087
6088     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6089
6090     my $volid_hash = {};
6091     foreach my $storeid (keys %$info) {
6092         foreach my $item (@{$info->{$storeid}}) {
6093             next if !($item->{volid} && $item->{size});
6094             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6095             $volid_hash->{$item->{volid}} = $item;
6096         }
6097     }
6098
6099     return $volid_hash;
6100 }
6101
6102 sub is_volume_in_use {
6103     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6104
6105     my $path = PVE::Storage::path($storecfg, $volid);
6106
6107     my $scan_config = sub {
6108         my ($cref, $snapname) = @_;
6109
6110         foreach my $key (keys %$cref) {
6111             my $value = $cref->{$key};
6112             if (is_valid_drivename($key)) {
6113                 next if $skip_drive && $key eq $skip_drive;
6114                 my $drive = parse_drive($key, $value);
6115                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6116                 return 1 if $volid eq $drive->{file};
6117                 if ($drive->{file} =~ m!^/!) {
6118                     return 1 if $drive->{file} eq $path;
6119                 } else {
6120                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6121                     next if !$storeid;
6122                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6123                     next if !$scfg;
6124                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6125                 }
6126             }
6127         }
6128
6129         return 0;
6130     };
6131
6132     return 1 if &$scan_config($conf);
6133
6134     undef $skip_drive;
6135
6136     foreach my $snapname (keys %{$conf->{snapshots}}) {
6137         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6138     }
6139
6140     return 0;
6141 }
6142
6143 sub update_disksize {
6144     my ($vmid, $conf, $volid_hash) = @_;
6145
6146     my $changes;
6147     my $prefix = "VM $vmid:";
6148
6149     # used and unused disks
6150     my $referenced = {};
6151
6152     # Note: it is allowed to define multiple storages with same path (alias), so
6153     # we need to check both 'volid' and real 'path' (two different volid can point
6154     # to the same path).
6155
6156     my $referencedpath = {};
6157
6158     # update size info
6159     foreach my $opt (keys %$conf) {
6160         if (is_valid_drivename($opt)) {
6161             my $drive = parse_drive($opt, $conf->{$opt});
6162             my $volid = $drive->{file};
6163             next if !$volid;
6164
6165             $referenced->{$volid} = 1;
6166             if ($volid_hash->{$volid} &&
6167                 (my $path = $volid_hash->{$volid}->{path})) {
6168                 $referencedpath->{$path} = 1;
6169             }
6170
6171             next if drive_is_cdrom($drive);
6172             next if !$volid_hash->{$volid};
6173
6174             $drive->{size} = $volid_hash->{$volid}->{size};
6175             my $new = print_drive($vmid, $drive);
6176             if ($new ne $conf->{$opt}) {
6177                 $changes = 1;
6178                 $conf->{$opt} = $new;
6179                 print "$prefix update disk '$opt' information.\n";
6180             }
6181         }
6182     }
6183
6184     # remove 'unusedX' entry if volume is used
6185     foreach my $opt (keys %$conf) {
6186         next if $opt !~ m/^unused\d+$/;
6187         my $volid = $conf->{$opt};
6188         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6189         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6190             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6191             $changes = 1;
6192             delete $conf->{$opt};
6193         }
6194
6195         $referenced->{$volid} = 1;
6196         $referencedpath->{$path} = 1 if $path;
6197     }
6198
6199     foreach my $volid (sort keys %$volid_hash) {
6200         next if $volid =~ m/vm-$vmid-state-/;
6201         next if $referenced->{$volid};
6202         my $path = $volid_hash->{$volid}->{path};
6203         next if !$path; # just to be sure
6204         next if $referencedpath->{$path};
6205         $changes = 1;
6206         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6207         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6208         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6209     }
6210
6211     return $changes;
6212 }
6213
6214 sub rescan {
6215     my ($vmid, $nolock, $dryrun) = @_;
6216
6217     my $cfg = PVE::Storage::config();
6218
6219     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6220     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6221     foreach my $stor (keys %{$cfg->{ids}}) {
6222         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6223     }
6224
6225     print "rescan volumes...\n";
6226     my $volid_hash = scan_volids($cfg, $vmid);
6227
6228     my $updatefn =  sub {
6229         my ($vmid) = @_;
6230
6231         my $conf = PVE::QemuConfig->load_config($vmid);
6232
6233         PVE::QemuConfig->check_lock($conf);
6234
6235         my $vm_volids = {};
6236         foreach my $volid (keys %$volid_hash) {
6237             my $info = $volid_hash->{$volid};
6238             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6239         }
6240
6241         my $changes = update_disksize($vmid, $conf, $vm_volids);
6242
6243         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6244     };
6245
6246     if (defined($vmid)) {
6247         if ($nolock) {
6248             &$updatefn($vmid);
6249         } else {
6250             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6251         }
6252     } else {
6253         my $vmlist = config_list();
6254         foreach my $vmid (keys %$vmlist) {
6255             if ($nolock) {
6256                 &$updatefn($vmid);
6257             } else {
6258                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6259             }
6260         }
6261     }
6262 }
6263
6264 sub restore_vma_archive {
6265     my ($archive, $vmid, $user, $opts, $comp) = @_;
6266
6267     my $readfrom = $archive;
6268
6269     my $cfg = PVE::Storage::config();
6270     my $commands = [];
6271     my $bwlimit = $opts->{bwlimit};
6272
6273     my $dbg_cmdstring = '';
6274     my $add_pipe = sub {
6275         my ($cmd) = @_;
6276         push @$commands, $cmd;
6277         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6278         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6279         $readfrom = '-';
6280     };
6281
6282     my $input = undef;
6283     if ($archive eq '-') {
6284         $input = '<&STDIN';
6285     } else {
6286         # If we use a backup from a PVE defined storage we also consider that
6287         # storage's rate limit:
6288         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6289         if (defined($volid)) {
6290             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6291             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6292             if ($readlimit) {
6293                 print STDERR "applying read rate limit: $readlimit\n";
6294                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6295                 $add_pipe->($cstream);
6296             }
6297         }
6298     }
6299
6300     if ($comp) {
6301         my $cmd;
6302         if ($comp eq 'gzip') {
6303             $cmd = ['zcat', $readfrom];
6304         } elsif ($comp eq 'lzop') {
6305             $cmd = ['lzop', '-d', '-c', $readfrom];
6306         } else {
6307             die "unknown compression method '$comp'\n";
6308         }
6309         $add_pipe->($cmd);
6310     }
6311
6312     my $tmpdir = "/var/tmp/vzdumptmp$$";
6313     rmtree $tmpdir;
6314
6315     # disable interrupts (always do cleanups)
6316     local $SIG{INT} =
6317         local $SIG{TERM} =
6318         local $SIG{QUIT} =
6319         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6320
6321     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6322     POSIX::mkfifo($mapfifo, 0600);
6323     my $fifofh;
6324
6325     my $openfifo = sub {
6326         open($fifofh, '>', $mapfifo) || die $!;
6327     };
6328
6329     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6330
6331     my $oldtimeout;
6332     my $timeout = 5;
6333
6334     my $devinfo = {};
6335
6336     my $rpcenv = PVE::RPCEnvironment::get();
6337
6338     my $conffile = PVE::QemuConfig->config_file($vmid);
6339     my $tmpfn = "$conffile.$$.tmp";
6340
6341     # Note: $oldconf is undef if VM does not exists
6342     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6343     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6344
6345     my %storage_limits;
6346
6347     my $print_devmap = sub {
6348         my $virtdev_hash = {};
6349
6350         my $cfgfn = "$tmpdir/qemu-server.conf";
6351
6352         # we can read the config - that is already extracted
6353         my $fh = IO::File->new($cfgfn, "r") ||
6354             "unable to read qemu-server.conf - $!\n";
6355
6356         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6357         if (-f $fwcfgfn) {
6358             my $pve_firewall_dir = '/etc/pve/firewall';
6359             mkdir $pve_firewall_dir; # make sure the dir exists
6360             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6361         }
6362
6363         while (defined(my $line = <$fh>)) {
6364             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6365                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6366                 die "archive does not contain data for drive '$virtdev'\n"
6367                     if !$devinfo->{$devname};
6368                 if (defined($opts->{storage})) {
6369                     $storeid = $opts->{storage} || 'local';
6370                 } elsif (!$storeid) {
6371                     $storeid = 'local';
6372                 }
6373                 $format = 'raw' if !$format;
6374                 $devinfo->{$devname}->{devname} = $devname;
6375                 $devinfo->{$devname}->{virtdev} = $virtdev;
6376                 $devinfo->{$devname}->{format} = $format;
6377                 $devinfo->{$devname}->{storeid} = $storeid;
6378
6379                 # check permission on storage
6380                 my $pool = $opts->{pool}; # todo: do we need that?
6381                 if ($user ne 'root@pam') {
6382                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6383                 }
6384
6385                 $storage_limits{$storeid} = $bwlimit;
6386
6387                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6388             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6389                 my $virtdev = $1;
6390                 my $drive = parse_drive($virtdev, $2);
6391                 if (drive_is_cloudinit($drive)) {
6392                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6393                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6394                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6395
6396                     my $d = {
6397                         format => $format,
6398                         storeid => $opts->{storage} // $storeid,
6399                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6400                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6401                         name => "vm-$vmid-cloudinit",
6402                         is_cloudinit => 1,
6403                     };
6404                     $virtdev_hash->{$virtdev} = $d;
6405                 }
6406             }
6407         }
6408
6409         foreach my $key (keys %storage_limits) {
6410             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6411             next if !$limit;
6412             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6413             $storage_limits{$key} = $limit * 1024;
6414         }
6415
6416         foreach my $devname (keys %$devinfo) {
6417             die "found no device mapping information for device '$devname'\n"
6418                 if !$devinfo->{$devname}->{virtdev};
6419         }
6420
6421         # create empty/temp config
6422         if ($oldconf) {
6423             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6424             foreach_drive($oldconf, sub {
6425                 my ($ds, $drive) = @_;
6426
6427                 return if drive_is_cdrom($drive, 1);
6428
6429                 my $volid = $drive->{file};
6430                 return if !$volid || $volid =~ m|^/|;
6431
6432                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6433                 return if !$path || !$owner || ($owner != $vmid);
6434
6435                 # Note: only delete disk we want to restore
6436                 # other volumes will become unused
6437                 if ($virtdev_hash->{$ds}) {
6438                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6439                     if (my $err = $@) {
6440                         warn $err;
6441                     }
6442                 }
6443             });
6444
6445             # delete vmstate files, after the restore we have no snapshots anymore
6446             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6447                 my $snap = $oldconf->{snapshots}->{$snapname};
6448                 if ($snap->{vmstate}) {
6449                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6450                     if (my $err = $@) {
6451                         warn $err;
6452                     }
6453                 }
6454             }
6455         }
6456
6457         my $map = {};
6458         foreach my $virtdev (sort keys %$virtdev_hash) {
6459             my $d = $virtdev_hash->{$virtdev};
6460             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6461             my $storeid = $d->{storeid};
6462             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6463
6464             my $map_opts = '';
6465             if (my $limit = $storage_limits{$storeid}) {
6466                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6467             }
6468
6469             # test if requested format is supported
6470             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6471             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6472             $d->{format} = $defFormat if !$supported;
6473
6474             my $name;
6475             if ($d->{is_cloudinit}) {
6476                 $name = $d->{name};
6477                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6478             }
6479
6480             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6481             print STDERR "new volume ID is '$volid'\n";
6482             $d->{volid} = $volid;
6483
6484             PVE::Storage::activate_volumes($cfg, [$volid]);
6485
6486             my $write_zeros = 1;
6487             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6488                 $write_zeros = 0;
6489             }
6490
6491             if (!$d->{is_cloudinit}) {
6492                 my $path = PVE::Storage::path($cfg, $volid);
6493
6494                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6495
6496                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6497             }
6498             $map->{$virtdev} = $volid;
6499         }
6500
6501         $fh->seek(0, 0) || die "seek failed - $!\n";
6502
6503         my $outfd = new IO::File ($tmpfn, "w") ||
6504             die "unable to write config for VM $vmid\n";
6505
6506         my $cookie = { netcount => 0 };
6507         while (defined(my $line = <$fh>)) {
6508             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6509         }
6510
6511         $fh->close();
6512         $outfd->close();
6513     };
6514
6515     eval {
6516         # enable interrupts
6517         local $SIG{INT} =
6518             local $SIG{TERM} =
6519             local $SIG{QUIT} =
6520             local $SIG{HUP} =
6521             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6522         local $SIG{ALRM} = sub { die "got timeout\n"; };
6523
6524         $oldtimeout = alarm($timeout);
6525
6526         my $parser = sub {
6527             my $line = shift;
6528
6529             print "$line\n";
6530
6531             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6532                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6533                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6534             } elsif ($line =~ m/^CTIME: /) {
6535                 # we correctly received the vma config, so we can disable
6536                 # the timeout now for disk allocation (set to 10 minutes, so
6537                 # that we always timeout if something goes wrong)
6538                 alarm(600);
6539                 &$print_devmap();
6540                 print $fifofh "done\n";
6541                 my $tmp = $oldtimeout || 0;
6542                 $oldtimeout = undef;
6543                 alarm($tmp);
6544                 close($fifofh);
6545             }
6546         };
6547
6548         print "restore vma archive: $dbg_cmdstring\n";
6549         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6550     };
6551     my $err = $@;
6552
6553     alarm($oldtimeout) if $oldtimeout;
6554
6555     my $vollist = [];
6556     foreach my $devname (keys %$devinfo) {
6557         my $volid = $devinfo->{$devname}->{volid};
6558         push @$vollist, $volid if $volid;
6559     }
6560
6561     PVE::Storage::deactivate_volumes($cfg, $vollist);
6562
6563     unlink $mapfifo;
6564
6565     if ($err) {
6566         rmtree $tmpdir;
6567         unlink $tmpfn;
6568
6569         foreach my $devname (keys %$devinfo) {
6570             my $volid = $devinfo->{$devname}->{volid};
6571             next if !$volid;
6572             eval {
6573                 if ($volid =~ m|^/|) {
6574                     unlink $volid || die 'unlink failed\n';
6575                 } else {
6576                     PVE::Storage::vdisk_free($cfg, $volid);
6577                 }
6578                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6579             };
6580             print STDERR "unable to cleanup '$volid' - $@" if $@;
6581         }
6582         die $err;
6583     }
6584
6585     rmtree $tmpdir;
6586
6587     rename($tmpfn, $conffile) ||
6588         die "unable to commit configuration file '$conffile'\n";
6589
6590     PVE::Cluster::cfs_update(); # make sure we read new file
6591
6592     eval { rescan($vmid, 1); };
6593     warn $@ if $@;
6594 }
6595
6596 sub restore_tar_archive {
6597     my ($archive, $vmid, $user, $opts) = @_;
6598
6599     if ($archive ne '-') {
6600         my $firstfile = tar_archive_read_firstfile($archive);
6601         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6602             if $firstfile ne 'qemu-server.conf';
6603     }
6604
6605     my $storecfg = PVE::Storage::config();
6606
6607     # destroy existing data - keep empty config
6608     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6609     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6610
6611     my $tocmd = "/usr/lib/qemu-server/qmextract";
6612
6613     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6614     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6615     $tocmd .= ' --prealloc' if $opts->{prealloc};
6616     $tocmd .= ' --info' if $opts->{info};
6617
6618     # tar option "xf" does not autodetect compression when read from STDIN,
6619     # so we pipe to zcat
6620     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6621         PVE::Tools::shellquote("--to-command=$tocmd");
6622
6623     my $tmpdir = "/var/tmp/vzdumptmp$$";
6624     mkpath $tmpdir;
6625
6626     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6627     local $ENV{VZDUMP_VMID} = $vmid;
6628     local $ENV{VZDUMP_USER} = $user;
6629
6630     my $conffile = PVE::QemuConfig->config_file($vmid);
6631     my $tmpfn = "$conffile.$$.tmp";
6632
6633     # disable interrupts (always do cleanups)
6634     local $SIG{INT} =
6635         local $SIG{TERM} =
6636         local $SIG{QUIT} =
6637         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6638
6639     eval {
6640         # enable interrupts
6641         local $SIG{INT} =
6642             local $SIG{TERM} =
6643             local $SIG{QUIT} =
6644             local $SIG{HUP} =
6645             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6646
6647         if ($archive eq '-') {
6648             print "extracting archive from STDIN\n";
6649             run_command($cmd, input => "<&STDIN");
6650         } else {
6651             print "extracting archive '$archive'\n";
6652             run_command($cmd);
6653         }
6654
6655         return if $opts->{info};
6656
6657         # read new mapping
6658         my $map = {};
6659         my $statfile = "$tmpdir/qmrestore.stat";
6660         if (my $fd = IO::File->new($statfile, "r")) {
6661             while (defined (my $line = <$fd>)) {
6662                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6663                     $map->{$1} = $2 if $1;
6664                 } else {
6665                     print STDERR "unable to parse line in statfile - $line\n";
6666                 }
6667             }
6668             $fd->close();
6669         }
6670
6671         my $confsrc = "$tmpdir/qemu-server.conf";
6672
6673         my $srcfd = new IO::File($confsrc, "r") ||
6674             die "unable to open file '$confsrc'\n";
6675
6676         my $outfd = new IO::File ($tmpfn, "w") ||
6677             die "unable to write config for VM $vmid\n";
6678
6679         my $cookie = { netcount => 0 };
6680         while (defined (my $line = <$srcfd>)) {
6681             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6682         }
6683
6684         $srcfd->close();
6685         $outfd->close();
6686     };
6687     my $err = $@;
6688
6689     if ($err) {
6690
6691         unlink $tmpfn;
6692
6693         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6694
6695         die $err;
6696     }
6697
6698     rmtree $tmpdir;
6699
6700     rename $tmpfn, $conffile ||
6701         die "unable to commit configuration file '$conffile'\n";
6702
6703     PVE::Cluster::cfs_update(); # make sure we read new file
6704
6705     eval { rescan($vmid, 1); };
6706     warn $@ if $@;
6707 };
6708
6709 sub foreach_storage_used_by_vm {
6710     my ($conf, $func) = @_;
6711
6712     my $sidhash = {};
6713
6714     foreach_drive($conf, sub {
6715         my ($ds, $drive) = @_;
6716         return if drive_is_cdrom($drive);
6717
6718         my $volid = $drive->{file};
6719
6720         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6721         $sidhash->{$sid} = $sid if $sid;
6722     });
6723
6724     foreach my $sid (sort keys %$sidhash) {
6725         &$func($sid);
6726     }
6727 }
6728
6729 sub do_snapshots_with_qemu {
6730     my ($storecfg, $volid) = @_;
6731
6732     my $storage_name = PVE::Storage::parse_volume_id($volid);
6733     my $scfg = $storecfg->{ids}->{$storage_name};
6734
6735     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6736         return 1;
6737     }
6738
6739     if ($volid =~ m/\.(qcow2|qed)$/){
6740         return 1;
6741     }
6742
6743     return undef;
6744 }
6745
6746 sub qga_check_running {
6747     my ($vmid, $nowarn) = @_;
6748
6749     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6750     if ($@) {
6751         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6752         return 0;
6753     }
6754     return 1;
6755 }
6756
6757 sub template_create {
6758     my ($vmid, $conf, $disk) = @_;
6759
6760     my $storecfg = PVE::Storage::config();
6761
6762     foreach_drive($conf, sub {
6763         my ($ds, $drive) = @_;
6764
6765         return if drive_is_cdrom($drive);
6766         return if $disk && $ds ne $disk;
6767
6768         my $volid = $drive->{file};
6769         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6770
6771         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6772         $drive->{file} = $voliddst;
6773         $conf->{$ds} = print_drive($vmid, $drive);
6774         PVE::QemuConfig->write_config($vmid, $conf);
6775     });
6776 }
6777
6778 sub convert_iscsi_path {
6779     my ($path) = @_;
6780
6781     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6782         my $portal = $1;
6783         my $target = $2;
6784         my $lun = $3;
6785
6786         my $initiator_name = get_initiator_name();
6787
6788         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6789                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6790     }
6791
6792     die "cannot convert iscsi path '$path', unkown format\n";
6793 }
6794
6795 sub qemu_img_convert {
6796     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6797
6798     my $storecfg = PVE::Storage::config();
6799     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6800     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6801
6802     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6803
6804     my $cachemode;
6805     my $src_path;
6806     my $src_is_iscsi = 0;
6807     my $src_format = 'raw';
6808
6809     if ($src_storeid) {
6810         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6811         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6812         $src_format = qemu_img_format($src_scfg, $src_volname);
6813         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6814         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6815         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6816     } elsif (-f $src_volid) {
6817         $src_path = $src_volid;
6818         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6819             $src_format = $1;
6820         }
6821     }
6822
6823     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6824
6825     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6826     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6827     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6828     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6829
6830     my $cmd = [];
6831     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6832     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6833     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6834     push @$cmd, '-T', $cachemode if defined($cachemode);
6835
6836     if ($src_is_iscsi) {
6837         push @$cmd, '--image-opts';
6838         $src_path = convert_iscsi_path($src_path);
6839     } else {
6840         push @$cmd, '-f', $src_format;
6841     }
6842
6843     if ($dst_is_iscsi) {
6844         push @$cmd, '--target-image-opts';
6845         $dst_path = convert_iscsi_path($dst_path);
6846     } else {
6847         push @$cmd, '-O', $dst_format;
6848     }
6849
6850     push @$cmd, $src_path;
6851
6852     if (!$dst_is_iscsi && $is_zero_initialized) {
6853         push @$cmd, "zeroinit:$dst_path";
6854     } else {
6855         push @$cmd, $dst_path;
6856     }
6857
6858     my $parser = sub {
6859         my $line = shift;
6860         if($line =~ m/\((\S+)\/100\%\)/){
6861             my $percent = $1;
6862             my $transferred = int($size * $percent / 100);
6863             my $remaining = $size - $transferred;
6864
6865             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6866         }
6867
6868     };
6869
6870     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6871     my $err = $@;
6872     die "copy failed: $err" if $err;
6873 }
6874
6875 sub qemu_img_format {
6876     my ($scfg, $volname) = @_;
6877
6878     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6879         return $1;
6880     } else {
6881         return "raw";
6882     }
6883 }
6884
6885 sub qemu_drive_mirror {
6886     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6887
6888     $jobs = {} if !$jobs;
6889
6890     my $qemu_target;
6891     my $format;
6892     $jobs->{"drive-$drive"} = {};
6893
6894     if ($dst_volid =~ /^nbd:/) {
6895         $qemu_target = $dst_volid;
6896         $format = "nbd";
6897     } else {
6898         my $storecfg = PVE::Storage::config();
6899         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6900
6901         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6902
6903         $format = qemu_img_format($dst_scfg, $dst_volname);
6904
6905         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6906
6907         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6908     }
6909
6910     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6911     $opts->{format} = $format if $format;
6912
6913     if (defined($bwlimit)) {
6914         $opts->{speed} = $bwlimit * 1024;
6915         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6916     } else {
6917         print "drive mirror is starting for drive-$drive\n";
6918     }
6919
6920     # if a job already runs for this device we get an error, catch it for cleanup
6921     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6922     if (my $err = $@) {
6923         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6924         warn "$@\n" if $@;
6925         die "mirroring error: $err\n";
6926     }
6927
6928     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6929 }
6930
6931 sub qemu_drive_mirror_monitor {
6932     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6933
6934     eval {
6935         my $err_complete = 0;
6936
6937         while (1) {
6938             die "storage migration timed out\n" if $err_complete > 300;
6939
6940             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6941
6942             my $running_mirror_jobs = {};
6943             foreach my $stat (@$stats) {
6944                 next if $stat->{type} ne 'mirror';
6945                 $running_mirror_jobs->{$stat->{device}} = $stat;
6946             }
6947
6948             my $readycounter = 0;
6949
6950             foreach my $job (keys %$jobs) {
6951
6952                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6953                     print "$job : finished\n";
6954                     delete $jobs->{$job};
6955                     next;
6956                 }
6957
6958                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6959
6960                 my $busy = $running_mirror_jobs->{$job}->{busy};
6961                 my $ready = $running_mirror_jobs->{$job}->{ready};
6962                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6963                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6964                     my $remaining = $total - $transferred;
6965                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6966
6967                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6968                 }
6969
6970                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6971             }
6972
6973             last if scalar(keys %$jobs) == 0;
6974
6975             if ($readycounter == scalar(keys %$jobs)) {
6976                 print "all mirroring jobs are ready \n";
6977                 last if $skipcomplete; #do the complete later
6978
6979                 if ($vmiddst && $vmiddst != $vmid) {
6980                     my $agent_running = $qga && qga_check_running($vmid);
6981                     if ($agent_running) {
6982                         print "freeze filesystem\n";
6983                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6984                     } else {
6985                         print "suspend vm\n";
6986                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6987                     }
6988
6989                     # if we clone a disk for a new target vm, we don't switch the disk
6990                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6991
6992                     if ($agent_running) {
6993                         print "unfreeze filesystem\n";
6994                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6995                     } else {
6996                         print "resume vm\n";
6997                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6998                     }
6999
7000                     last;
7001                 } else {
7002
7003                     foreach my $job (keys %$jobs) {
7004                         # try to switch the disk if source and destination are on the same guest
7005                         print "$job: Completing block job...\n";
7006
7007                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7008                         if ($@ =~ m/cannot be completed/) {
7009                             print "$job: Block job cannot be completed, try again.\n";
7010                             $err_complete++;
7011                         }else {
7012                             print "$job: Completed successfully.\n";
7013                             $jobs->{$job}->{complete} = 1;
7014                         }
7015                     }
7016                 }
7017             }
7018             sleep 1;
7019         }
7020     };
7021     my $err = $@;
7022
7023     if ($err) {
7024         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7025         die "mirroring error: $err";
7026     }
7027
7028 }
7029
7030 sub qemu_blockjobs_cancel {
7031     my ($vmid, $jobs) = @_;
7032
7033     foreach my $job (keys %$jobs) {
7034         print "$job: Cancelling block job\n";
7035         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7036         $jobs->{$job}->{cancel} = 1;
7037     }
7038
7039     while (1) {
7040         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7041
7042         my $running_jobs = {};
7043         foreach my $stat (@$stats) {
7044             $running_jobs->{$stat->{device}} = $stat;
7045         }
7046
7047         foreach my $job (keys %$jobs) {
7048
7049             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7050                 print "$job: Done.\n";
7051                 delete $jobs->{$job};
7052             }
7053         }
7054
7055         last if scalar(keys %$jobs) == 0;
7056
7057         sleep 1;
7058     }
7059 }
7060
7061 sub clone_disk {
7062     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7063         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7064
7065     my $newvolid;
7066
7067     if (!$full) {
7068         print "create linked clone of drive $drivename ($drive->{file})\n";
7069         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7070         push @$newvollist, $newvolid;
7071     } else {
7072
7073         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7074         $storeid = $storage if $storage;
7075
7076         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7077         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7078
7079         print "create full clone of drive $drivename ($drive->{file})\n";
7080         my $name = undef;
7081         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7082         push @$newvollist, $newvolid;
7083
7084         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7085
7086         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7087         if (!$running || $snapname) {
7088             # TODO: handle bwlimits
7089             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7090         } else {
7091
7092             my $kvmver = get_running_qemu_version ($vmid);
7093             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7094                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7095                     if $drive->{iothread};
7096             }
7097
7098             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7099         }
7100     }
7101
7102     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7103
7104     my $disk = $drive;
7105     $disk->{format} = undef;
7106     $disk->{file} = $newvolid;
7107     $disk->{size} = $size;
7108
7109     return $disk;
7110 }
7111
7112 # this only works if VM is running
7113 sub get_current_qemu_machine {
7114     my ($vmid) = @_;
7115
7116     my $cmd = { execute => 'query-machines', arguments => {} };
7117     my $res = vm_qmp_command($vmid, $cmd);
7118
7119     my ($current, $default);
7120     foreach my $e (@$res) {
7121         $default = $e->{name} if $e->{'is-default'};
7122         $current = $e->{name} if $e->{'is-current'};
7123     }
7124
7125     # fallback to the default machine if current is not supported by qemu
7126     return $current || $default || 'pc';
7127 }
7128
7129 sub get_running_qemu_version {
7130     my ($vmid) = @_;
7131     my $cmd = { execute => 'query-version', arguments => {} };
7132     my $res = vm_qmp_command($vmid, $cmd);
7133     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7134 }
7135
7136 sub qemu_machine_feature_enabled {
7137     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7138
7139     my $current_major;
7140     my $current_minor;
7141
7142     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7143
7144         $current_major = $3;
7145         $current_minor = $4;
7146
7147     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7148
7149         $current_major = $1;
7150         $current_minor = $2;
7151     }
7152
7153     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7154 }
7155
7156 # gets in pairs the versions you want to compares, i.e.:
7157 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7158 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7159 sub version_cmp {
7160     my @versions = @_;
7161
7162     my $size = scalar(@versions);
7163
7164     return 0 if $size == 0;
7165     die "cannot compare odd count of versions" if $size & 1;
7166
7167     for (my $i = 0; $i < $size; $i += 2) {
7168         my ($a, $b) = splice(@versions, 0, 2);
7169         $a //= 0;
7170         $b //= 0;
7171
7172         return 1 if $a > $b;
7173         return -1 if $a < $b;
7174     }
7175     return 0;
7176 }
7177
7178 # dies if a) VM not running or not exisiting b) Version query failed
7179 # So, any defined return value is valid, any invalid state can be caught by eval
7180 sub runs_at_least_qemu_version {
7181     my ($vmid, $major, $minor, $extra) = @_;
7182
7183     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7184     die "could not query currently running version for VM $vmid\n" if !defined($v);
7185     $v = $v->{qemu};
7186
7187     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7188 }
7189
7190 sub qemu_machine_pxe {
7191     my ($vmid, $conf) = @_;
7192
7193     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7194
7195     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7196         $machine .= '.pxe';
7197     }
7198
7199     return $machine;
7200 }
7201
7202 sub qemu_use_old_bios_files {
7203     my ($machine_type) = @_;
7204
7205     return if !$machine_type;
7206
7207     my $use_old_bios_files = undef;
7208
7209     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7210         $machine_type = $1;
7211         $use_old_bios_files = 1;
7212     } else {
7213         my $kvmver = kvm_user_version();
7214         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7215         # load new efi bios files on migration. So this hack is required to allow
7216         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7217         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7218         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7219     }
7220
7221     return ($use_old_bios_files, $machine_type);
7222 }
7223
7224 sub create_efidisk($$$$$) {
7225     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7226
7227     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7228     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7229
7230     my $vars_size_b = -s $ovmf_vars;
7231     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7232     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7233     PVE::Storage::activate_volumes($storecfg, [$volid]);
7234
7235     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7236
7237     return ($volid, $vars_size);
7238 }
7239
7240 sub vm_iothreads_list {
7241     my ($vmid) = @_;
7242
7243     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7244
7245     my $iothreads = {};
7246     foreach my $iothread (@$res) {
7247         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7248     }
7249
7250     return $iothreads;
7251 }
7252
7253 sub scsihw_infos {
7254     my ($conf, $drive) = @_;
7255
7256     my $maxdev = 0;
7257
7258     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7259         $maxdev = 7;
7260     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7261         $maxdev = 1;
7262     } else {
7263         $maxdev = 256;
7264     }
7265
7266     my $controller = int($drive->{index} / $maxdev);
7267     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7268
7269     return ($maxdev, $controller, $controller_prefix);
7270 }
7271
7272 sub add_hyperv_enlightenments {
7273     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7274
7275     return if $winversion < 6;
7276     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7277
7278     if ($gpu_passthrough || defined($hv_vendor_id)) {
7279         $hv_vendor_id //= 'proxmox';
7280         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7281     }
7282
7283     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7284         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7285         push @$cpuFlags , 'hv_vapic';
7286         push @$cpuFlags , 'hv_time';
7287     } else {
7288         push @$cpuFlags , 'hv_spinlocks=0xffff';
7289     }
7290
7291     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7292         push @$cpuFlags , 'hv_reset';
7293         push @$cpuFlags , 'hv_vpindex';
7294         push @$cpuFlags , 'hv_runtime';
7295     }
7296
7297     if ($winversion >= 7) {
7298         push @$cpuFlags , 'hv_relaxed';
7299
7300         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7301             push @$cpuFlags , 'hv_synic';
7302             push @$cpuFlags , 'hv_stimer';
7303         }
7304
7305         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7306             push @$cpuFlags , 'hv_ipi';
7307         }
7308     }
7309 }
7310
7311 sub windows_version {
7312     my ($ostype) = @_;
7313
7314     return 0 if !$ostype;
7315
7316     my $winversion = 0;
7317
7318     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7319         $winversion = 5;
7320     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7321         $winversion = 6;
7322     } elsif ($ostype =~ m/^win(\d+)$/) {
7323         $winversion = $1;
7324     }
7325
7326     return $winversion;
7327 }
7328
7329 sub resolve_dst_disk_format {
7330         my ($storecfg, $storeid, $src_volname, $format) = @_;
7331         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7332
7333         if (!$format) {
7334             # if no target format is specified, use the source disk format as hint
7335             if ($src_volname) {
7336                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7337                 $format = qemu_img_format($scfg, $src_volname);
7338             } else {
7339                 return $defFormat;
7340             }
7341         }
7342
7343         # test if requested format is supported - else use default
7344         my $supported = grep { $_ eq $format } @$validFormats;
7345         $format = $defFormat if !$supported;
7346         return $format;
7347 }
7348
7349 sub resolve_first_disk {
7350     my $conf = shift;
7351     my @disks = PVE::QemuServer::valid_drive_names();
7352     my $firstdisk;
7353     foreach my $ds (reverse @disks) {
7354         next if !$conf->{$ds};
7355         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7356         next if PVE::QemuServer::drive_is_cdrom($disk);
7357         $firstdisk = $ds;
7358     }
7359     return $firstdisk;
7360 }
7361
7362 sub generate_uuid {
7363     my ($uuid, $uuid_str);
7364     UUID::generate($uuid);
7365     UUID::unparse($uuid, $uuid_str);
7366     return $uuid_str;
7367 }
7368
7369 sub generate_smbios1_uuid {
7370     return "uuid=".generate_uuid();
7371 }
7372
7373 sub nbd_stop {
7374     my ($vmid) = @_;
7375
7376     vm_mon_cmd($vmid, 'nbd-server-stop');
7377 }
7378
7379 sub create_reboot_request {
7380     my ($vmid) = @_;
7381     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7382         or die "failed to create reboot trigger file: $!\n";
7383     close($fh);
7384 }
7385
7386 sub clear_reboot_request {
7387     my ($vmid) = @_;
7388     my $path = "/run/qemu-server/$vmid.reboot";
7389     my $res = 0;
7390
7391     $res = unlink($path);
7392     die "could not remove reboot request for $vmid: $!"
7393         if !$res && $! != POSIX::ENOENT;
7394
7395     return $res;
7396 }
7397
7398 # bash completion helper
7399
7400 sub complete_backup_archives {
7401     my ($cmdname, $pname, $cvalue) = @_;
7402
7403     my $cfg = PVE::Storage::config();
7404
7405     my $storeid;
7406
7407     if ($cvalue =~ m/^([^:]+):/) {
7408         $storeid = $1;
7409     }
7410
7411     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7412
7413     my $res = [];
7414     foreach my $id (keys %$data) {
7415         foreach my $item (@{$data->{$id}}) {
7416             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7417             push @$res, $item->{volid} if defined($item->{volid});
7418         }
7419     }
7420
7421     return $res;
7422 }
7423
7424 my $complete_vmid_full = sub {
7425     my ($running) = @_;
7426
7427     my $idlist = vmstatus();
7428
7429     my $res = [];
7430
7431     foreach my $id (keys %$idlist) {
7432         my $d = $idlist->{$id};
7433         if (defined($running)) {
7434             next if $d->{template};
7435             next if $running && $d->{status} ne 'running';
7436             next if !$running && $d->{status} eq 'running';
7437         }
7438         push @$res, $id;
7439
7440     }
7441     return $res;
7442 };
7443
7444 sub complete_vmid {
7445     return &$complete_vmid_full();
7446 }
7447
7448 sub complete_vmid_stopped {
7449     return &$complete_vmid_full(0);
7450 }
7451
7452 sub complete_vmid_running {
7453     return &$complete_vmid_full(1);
7454 }
7455
7456 sub complete_storage {
7457
7458     my $cfg = PVE::Storage::config();
7459     my $ids = $cfg->{ids};
7460
7461     my $res = [];
7462     foreach my $sid (keys %$ids) {
7463         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7464         next if !$ids->{$sid}->{content}->{images};
7465         push @$res, $sid;
7466     }
7467
7468     return $res;
7469 }
7470
7471 1;