PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             if($path =~ m/^iscsi\:\/\//){
1902                 $devicetype = 'generic';
1903             }
1904         }
1905
1906         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1907             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1908         } else {
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         }
1911
1912         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1913             $device .= ",rotation_rate=1";
1914         }
1915         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1916
1917     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1918         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1919         my $controller = int($drive->{index} / $maxdev);
1920         my $unit = $drive->{index} % $maxdev;
1921         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1922
1923         $device = "ide-$devicetype";
1924         if ($drive->{interface} eq 'ide') {
1925             $device .= ",bus=ide.$controller,unit=$unit";
1926         } else {
1927             $device .= ",bus=ahci$controller.$unit";
1928         }
1929         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1930
1931         if ($devicetype eq 'hd') {
1932             if (my $model = $drive->{model}) {
1933                 $model = URI::Escape::uri_unescape($model);
1934                 $device .= ",model=$model";
1935             }
1936             if ($drive->{ssd}) {
1937                 $device .= ",rotation_rate=1";
1938             }
1939         }
1940         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1941     } elsif ($drive->{interface} eq 'usb') {
1942         die "implement me";
1943         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1944     } else {
1945         die "unsupported interface type";
1946     }
1947
1948     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1949
1950     if (my $serial = $drive->{serial}) {
1951         $serial = URI::Escape::uri_unescape($serial);
1952         $device .= ",serial=$serial";
1953     }
1954
1955
1956     return $device;
1957 }
1958
1959 sub get_initiator_name {
1960     my $initiator;
1961
1962     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1963     while (defined(my $line = <$fh>)) {
1964         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1965         $initiator = $1;
1966         last;
1967     }
1968     $fh->close();
1969
1970     return $initiator;
1971 }
1972
1973 sub print_drive_full {
1974     my ($storecfg, $vmid, $drive) = @_;
1975
1976     my $path;
1977     my $volid = $drive->{file};
1978     my $format;
1979
1980     if (drive_is_cdrom($drive)) {
1981         $path = get_iso_path($storecfg, $vmid, $volid);
1982     } else {
1983         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1984         if ($storeid) {
1985             $path = PVE::Storage::path($storecfg, $volid);
1986             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1987             $format = qemu_img_format($scfg, $volname);
1988         } else {
1989             $path = $volid;
1990             $format = "raw";
1991         }
1992    }
1993
1994     my $opts = '';
1995     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1996     foreach my $o (@qemu_drive_options) {
1997         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1998     }
1999
2000     # snapshot only accepts on|off
2001     if (defined($drive->{snapshot})) {
2002         my $v = $drive->{snapshot} ? 'on' : 'off';
2003         $opts .= ",snapshot=$v";
2004     }
2005
2006     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2007         my ($dir, $qmpname) = @$type;
2008         if (my $v = $drive->{"mbps$dir"}) {
2009             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2010         }
2011         if (my $v = $drive->{"mbps${dir}_max"}) {
2012             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2013         }
2014         if (my $v = $drive->{"bps${dir}_max_length"}) {
2015             $opts .= ",throttling.bps$qmpname-max-length=$v";
2016         }
2017         if (my $v = $drive->{"iops${dir}"}) {
2018             $opts .= ",throttling.iops$qmpname=$v";
2019         }
2020         if (my $v = $drive->{"iops${dir}_max"}) {
2021             $opts .= ",throttling.iops$qmpname-max=$v";
2022         }
2023         if (my $v = $drive->{"iops${dir}_max_length"}) {
2024             $opts .= ",throttling.iops$qmpname-max-length=$v";
2025         }
2026     }
2027
2028     $opts .= ",format=$format" if $format && !$drive->{format};
2029
2030     my $cache_direct = 0;
2031
2032     if (my $cache = $drive->{cache}) {
2033         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2034     } elsif (!drive_is_cdrom($drive)) {
2035         $opts .= ",cache=none";
2036         $cache_direct = 1;
2037     }
2038
2039     # aio native works only with O_DIRECT
2040     if (!$drive->{aio}) {
2041         if($cache_direct) {
2042             $opts .= ",aio=native";
2043         } else {
2044             $opts .= ",aio=threads";
2045         }
2046     }
2047
2048     if (!drive_is_cdrom($drive)) {
2049         my $detectzeroes;
2050         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2051             $detectzeroes = 'off';
2052         } elsif ($drive->{discard}) {
2053             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2054         } else {
2055             # This used to be our default with discard not being specified:
2056             $detectzeroes = 'on';
2057         }
2058         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2059     }
2060
2061     my $pathinfo = $path ? "file=$path," : '';
2062
2063     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2064 }
2065
2066 sub print_netdevice_full {
2067     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2068
2069     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2070
2071     my $device = $net->{model};
2072     if ($net->{model} eq 'virtio') {
2073          $device = 'virtio-net-pci';
2074      };
2075
2076     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2077     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2078     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2079         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2080         my $vectors = $net->{queues} * 2 + 2;
2081         $tmpstr .= ",vectors=$vectors,mq=on";
2082     }
2083     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2084
2085     if ($use_old_bios_files) {
2086         my $romfile;
2087         if ($device eq 'virtio-net-pci') {
2088             $romfile = 'pxe-virtio.rom';
2089         } elsif ($device eq 'e1000') {
2090             $romfile = 'pxe-e1000.rom';
2091         } elsif ($device eq 'ne2k') {
2092             $romfile = 'pxe-ne2k_pci.rom';
2093         } elsif ($device eq 'pcnet') {
2094             $romfile = 'pxe-pcnet.rom';
2095         } elsif ($device eq 'rtl8139') {
2096             $romfile = 'pxe-rtl8139.rom';
2097         }
2098         $tmpstr .= ",romfile=$romfile" if $romfile;
2099     }
2100
2101     return $tmpstr;
2102 }
2103
2104 sub print_netdev_full {
2105     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2106
2107     my $i = '';
2108     if ($netid =~ m/^net(\d+)$/) {
2109         $i = int($1);
2110     }
2111
2112     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2113
2114     my $ifname = "tap${vmid}i$i";
2115
2116     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2117     die "interface name '$ifname' is too long (max 15 character)\n"
2118         if length($ifname) >= 16;
2119
2120     my $vhostparam = '';
2121     if (is_native($arch)) {
2122         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2123     }
2124
2125     my $vmname = $conf->{name} || "vm$vmid";
2126
2127     my $netdev = "";
2128     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2129
2130     if ($net->{bridge}) {
2131         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2132     } else {
2133         $netdev = "type=user,id=$netid,hostname=$vmname";
2134     }
2135
2136     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2137
2138     return $netdev;
2139 }
2140
2141
2142 sub print_cpu_device {
2143     my ($conf, $id) = @_;
2144
2145     my $kvm = $conf->{kvm} // 1;
2146     my $cpu = $kvm ? "kvm64" : "qemu64";
2147     if (my $cputype = $conf->{cpu}) {
2148         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2149             or die "Cannot parse cpu description: $cputype\n";
2150         $cpu = $cpuconf->{cputype};
2151     }
2152
2153     my $cores = $conf->{cores} || 1;
2154
2155     my $current_core = ($id - 1) % $cores;
2156     my $current_socket = int(($id - 1 - $current_core)/$cores);
2157
2158     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2159 }
2160
2161 my $vga_map = {
2162     'cirrus' => 'cirrus-vga',
2163     'std' => 'VGA',
2164     'vmware' => 'vmware-svga',
2165     'virtio' => 'virtio-vga',
2166 };
2167
2168 sub print_vga_device {
2169     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2170
2171     my $type = $vga_map->{$vga->{type}};
2172     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2173         $type = 'virtio-gpu';
2174     }
2175     my $vgamem_mb = $vga->{memory};
2176     if ($qxlnum) {
2177         $type = $id ? 'qxl' : 'qxl-vga';
2178     }
2179     die "no devicetype for $vga->{type}\n" if !$type;
2180
2181     my $memory = "";
2182     if ($vgamem_mb) {
2183         if ($vga->{type} eq 'virtio') {
2184             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2185             $memory = ",max_hostmem=$bytes";
2186         } elsif ($qxlnum) {
2187             # from https://www.spice-space.org/multiple-monitors.html
2188             $memory = ",vgamem_mb=$vga->{memory}";
2189             my $ram = $vgamem_mb * 4;
2190             my $vram = $vgamem_mb * 2;
2191             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2192         } else {
2193             $memory = ",vgamem_mb=$vga->{memory}";
2194         }
2195     } elsif ($qxlnum && $id) {
2196         $memory = ",ram_size=67108864,vram_size=33554432";
2197     }
2198
2199     my $q35 = machine_type_is_q35($conf);
2200     my $vgaid = "vga" . ($id // '');
2201     my $pciaddr;
2202
2203     if ($q35 && $vgaid eq 'vga') {
2204         # the first display uses pcie.0 bus on q35 machines
2205         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2206     } else {
2207         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2208     }
2209
2210     return "$type,id=${vgaid}${memory}${pciaddr}";
2211 }
2212
2213 sub drive_is_cloudinit {
2214     my ($drive) = @_;
2215     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2216 }
2217
2218 sub drive_is_cdrom {
2219     my ($drive, $exclude_cloudinit) = @_;
2220
2221     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2222
2223     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2224
2225 }
2226
2227 sub parse_number_sets {
2228     my ($set) = @_;
2229     my $res = [];
2230     foreach my $part (split(/;/, $set)) {
2231         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2232             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2233             push @$res, [ $1, $2 ];
2234         } else {
2235             die "invalid range: $part\n";
2236         }
2237     }
2238     return $res;
2239 }
2240
2241 sub parse_numa {
2242     my ($data) = @_;
2243
2244     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2245     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2246     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2247     return $res;
2248 }
2249
2250 sub parse_hostpci {
2251     my ($value) = @_;
2252
2253     return undef if !$value;
2254
2255     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2256
2257     my @idlist = split(/;/, $res->{host});
2258     delete $res->{host};
2259     foreach my $id (@idlist) {
2260         if ($id =~ m/\./) { # full id 00:00.1
2261             push @{$res->{pciid}}, {
2262                 id => $id,
2263             };
2264         } else { # partial id 00:00
2265             $res->{pciid} = PVE::SysFSTools::lspci($id);
2266         }
2267     }
2268     return $res;
2269 }
2270
2271 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2272 sub parse_net {
2273     my ($data) = @_;
2274
2275     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2276     if ($@) {
2277         warn $@;
2278         return undef;
2279     }
2280     if (!defined($res->{macaddr})) {
2281         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2282         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2283     }
2284     return $res;
2285 }
2286
2287 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2288 sub parse_ipconfig {
2289     my ($data) = @_;
2290
2291     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2292     if ($@) {
2293         warn $@;
2294         return undef;
2295     }
2296
2297     if ($res->{gw} && !$res->{ip}) {
2298         warn 'gateway specified without specifying an IP address';
2299         return undef;
2300     }
2301     if ($res->{gw6} && !$res->{ip6}) {
2302         warn 'IPv6 gateway specified without specifying an IPv6 address';
2303         return undef;
2304     }
2305     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2306         warn 'gateway specified together with DHCP';
2307         return undef;
2308     }
2309     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2310         # gw6 + auto/dhcp
2311         warn "IPv6 gateway specified together with $res->{ip6} address";
2312         return undef;
2313     }
2314
2315     if (!$res->{ip} && !$res->{ip6}) {
2316         return { ip => 'dhcp', ip6 => 'dhcp' };
2317     }
2318
2319     return $res;
2320 }
2321
2322 sub print_net {
2323     my $net = shift;
2324
2325     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2326 }
2327
2328 sub add_random_macs {
2329     my ($settings) = @_;
2330
2331     foreach my $opt (keys %$settings) {
2332         next if $opt !~ m/^net(\d+)$/;
2333         my $net = parse_net($settings->{$opt});
2334         next if !$net;
2335         $settings->{$opt} = print_net($net);
2336     }
2337 }
2338
2339 sub vm_is_volid_owner {
2340     my ($storecfg, $vmid, $volid) = @_;
2341
2342     if ($volid !~  m|^/|) {
2343         my ($path, $owner);
2344         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2345         if ($owner && ($owner == $vmid)) {
2346             return 1;
2347         }
2348     }
2349
2350     return undef;
2351 }
2352
2353 sub vmconfig_register_unused_drive {
2354     my ($storecfg, $vmid, $conf, $drive) = @_;
2355
2356     if (drive_is_cloudinit($drive)) {
2357         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2358         warn $@ if $@;
2359     } elsif (!drive_is_cdrom($drive)) {
2360         my $volid = $drive->{file};
2361         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2362             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2363         }
2364     }
2365 }
2366
2367 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2368 my $smbios1_fmt = {
2369     uuid => {
2370         type => 'string',
2371         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2372         format_description => 'UUID',
2373         description => "Set SMBIOS1 UUID.",
2374         optional => 1,
2375     },
2376     version => {
2377         type => 'string',
2378         pattern => '[A-Za-z0-9+\/]+={0,2}',
2379         format_description => 'Base64 encoded string',
2380         description => "Set SMBIOS1 version.",
2381         optional => 1,
2382     },
2383     serial => {
2384         type => 'string',
2385         pattern => '[A-Za-z0-9+\/]+={0,2}',
2386         format_description => 'Base64 encoded string',
2387         description => "Set SMBIOS1 serial number.",
2388         optional => 1,
2389     },
2390     manufacturer => {
2391         type => 'string',
2392         pattern => '[A-Za-z0-9+\/]+={0,2}',
2393         format_description => 'Base64 encoded string',
2394         description => "Set SMBIOS1 manufacturer.",
2395         optional => 1,
2396     },
2397     product => {
2398         type => 'string',
2399         pattern => '[A-Za-z0-9+\/]+={0,2}',
2400         format_description => 'Base64 encoded string',
2401         description => "Set SMBIOS1 product ID.",
2402         optional => 1,
2403     },
2404     sku => {
2405         type => 'string',
2406         pattern => '[A-Za-z0-9+\/]+={0,2}',
2407         format_description => 'Base64 encoded string',
2408         description => "Set SMBIOS1 SKU string.",
2409         optional => 1,
2410     },
2411     family => {
2412         type => 'string',
2413         pattern => '[A-Za-z0-9+\/]+={0,2}',
2414         format_description => 'Base64 encoded string',
2415         description => "Set SMBIOS1 family string.",
2416         optional => 1,
2417     },
2418     base64 => {
2419         type => 'boolean',
2420         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2421         optional => 1,
2422     },
2423 };
2424
2425 sub parse_smbios1 {
2426     my ($data) = @_;
2427
2428     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2429     warn $@ if $@;
2430     return $res;
2431 }
2432
2433 sub print_smbios1 {
2434     my ($smbios1) = @_;
2435     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2436 }
2437
2438 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2439
2440 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2441 sub verify_bootdisk {
2442     my ($value, $noerr) = @_;
2443
2444     return $value if is_valid_drivename($value);
2445
2446     return undef if $noerr;
2447
2448     die "invalid boot disk '$value'\n";
2449 }
2450
2451 sub parse_watchdog {
2452     my ($value) = @_;
2453
2454     return undef if !$value;
2455
2456     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2457     warn $@ if $@;
2458     return $res;
2459 }
2460
2461 sub parse_guest_agent {
2462     my ($value) = @_;
2463
2464     return {} if !defined($value->{agent});
2465
2466     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2467     warn $@ if $@;
2468
2469     # if the agent is disabled ignore the other potentially set properties
2470     return {} if !$res->{enabled};
2471     return $res;
2472 }
2473
2474 sub parse_vga {
2475     my ($value) = @_;
2476
2477     return {} if !$value;
2478     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2479     warn $@ if $@;
2480     return $res;
2481 }
2482
2483 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2484 sub verify_usb_device {
2485     my ($value, $noerr) = @_;
2486
2487     return $value if parse_usb_device($value);
2488
2489     return undef if $noerr;
2490
2491     die "unable to parse usb device\n";
2492 }
2493
2494 # add JSON properties for create and set function
2495 sub json_config_properties {
2496     my $prop = shift;
2497
2498     foreach my $opt (keys %$confdesc) {
2499         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2500         $prop->{$opt} = $confdesc->{$opt};
2501     }
2502
2503     return $prop;
2504 }
2505
2506 # return copy of $confdesc_cloudinit to generate documentation
2507 sub cloudinit_config_properties {
2508
2509     return dclone($confdesc_cloudinit);
2510 }
2511
2512 sub check_type {
2513     my ($key, $value) = @_;
2514
2515     die "unknown setting '$key'\n" if !$confdesc->{$key};
2516
2517     my $type = $confdesc->{$key}->{type};
2518
2519     if (!defined($value)) {
2520         die "got undefined value\n";
2521     }
2522
2523     if ($value =~ m/[\n\r]/) {
2524         die "property contains a line feed\n";
2525     }
2526
2527     if ($type eq 'boolean') {
2528         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2529         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2530         die "type check ('boolean') failed - got '$value'\n";
2531     } elsif ($type eq 'integer') {
2532         return int($1) if $value =~ m/^(\d+)$/;
2533         die "type check ('integer') failed - got '$value'\n";
2534     } elsif ($type eq 'number') {
2535         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2536         die "type check ('number') failed - got '$value'\n";
2537     } elsif ($type eq 'string') {
2538         if (my $fmt = $confdesc->{$key}->{format}) {
2539             PVE::JSONSchema::check_format($fmt, $value);
2540             return $value;
2541         }
2542         $value =~ s/^\"(.*)\"$/$1/;
2543         return $value;
2544     } else {
2545         die "internal error"
2546     }
2547 }
2548
2549 sub touch_config {
2550     my ($vmid) = @_;
2551
2552     my $conf = PVE::QemuConfig->config_file($vmid);
2553     utime undef, undef, $conf;
2554 }
2555
2556 sub destroy_vm {
2557     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2558
2559     my $conf = PVE::QemuConfig->load_config($vmid);
2560
2561     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2562
2563     if ($conf->{template}) {
2564         # check if any base image is still used by a linked clone
2565         foreach_drive($conf, sub {
2566                 my ($ds, $drive) = @_;
2567
2568                 return if drive_is_cdrom($drive);
2569
2570                 my $volid = $drive->{file};
2571
2572                 return if !$volid || $volid =~ m|^/|;
2573
2574                 die "base volume '$volid' is still in use by linked cloned\n"
2575                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2576
2577         });
2578     }
2579
2580     # only remove disks owned by this VM
2581     foreach_drive($conf, sub {
2582         my ($ds, $drive) = @_;
2583
2584         return if drive_is_cdrom($drive, 1);
2585
2586         my $volid = $drive->{file};
2587
2588         return if !$volid || $volid =~ m|^/|;
2589
2590         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2591         return if !$path || !$owner || ($owner != $vmid);
2592
2593         eval {
2594             PVE::Storage::vdisk_free($storecfg, $volid);
2595         };
2596         warn "Could not remove disk '$volid', check manually: $@" if $@;
2597
2598     });
2599
2600     if ($keep_empty_config) {
2601         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2602     } else {
2603         PVE::QemuConfig->destroy_config($vmid);
2604     }
2605
2606     # also remove unused disk
2607     eval {
2608         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2609
2610         eval {
2611             PVE::Storage::foreach_volid($dl, sub {
2612                 my ($volid, $sid, $volname, $d) = @_;
2613                 PVE::Storage::vdisk_free($storecfg, $volid);
2614             });
2615         };
2616         warn $@ if $@;
2617
2618     };
2619     warn $@ if $@;
2620 }
2621
2622 sub parse_vm_config {
2623     my ($filename, $raw) = @_;
2624
2625     return undef if !defined($raw);
2626
2627     my $res = {
2628         digest => Digest::SHA::sha1_hex($raw),
2629         snapshots => {},
2630         pending => {},
2631     };
2632
2633     $filename =~ m|/qemu-server/(\d+)\.conf$|
2634         || die "got strange filename '$filename'";
2635
2636     my $vmid = $1;
2637
2638     my $conf = $res;
2639     my $descr;
2640     my $section = '';
2641
2642     my @lines = split(/\n/, $raw);
2643     foreach my $line (@lines) {
2644         next if $line =~ m/^\s*$/;
2645
2646         if ($line =~ m/^\[PENDING\]\s*$/i) {
2647             $section = 'pending';
2648             if (defined($descr)) {
2649                 $descr =~ s/\s+$//;
2650                 $conf->{description} = $descr;
2651             }
2652             $descr = undef;
2653             $conf = $res->{$section} = {};
2654             next;
2655
2656         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2657             $section = $1;
2658             if (defined($descr)) {
2659                 $descr =~ s/\s+$//;
2660                 $conf->{description} = $descr;
2661             }
2662             $descr = undef;
2663             $conf = $res->{snapshots}->{$section} = {};
2664             next;
2665         }
2666
2667         if ($line =~ m/^\#(.*)\s*$/) {
2668             $descr = '' if !defined($descr);
2669             $descr .= PVE::Tools::decode_text($1) . "\n";
2670             next;
2671         }
2672
2673         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2674             $descr = '' if !defined($descr);
2675             $descr .= PVE::Tools::decode_text($2);
2676         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2677             $conf->{snapstate} = $1;
2678         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2679             my $key = $1;
2680             my $value = $2;
2681             $conf->{$key} = $value;
2682         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2683             my $value = $1;
2684             if ($section eq 'pending') {
2685                 $conf->{delete} = $value; # we parse this later
2686             } else {
2687                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2688             }
2689         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2690             my $key = $1;
2691             my $value = $2;
2692             eval { $value = check_type($key, $value); };
2693             if ($@) {
2694                 warn "vm $vmid - unable to parse value of '$key' - $@";
2695             } else {
2696                 $key = 'ide2' if $key eq 'cdrom';
2697                 my $fmt = $confdesc->{$key}->{format};
2698                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2699                     my $v = parse_drive($key, $value);
2700                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2701                         $v->{file} = $volid;
2702                         $value = print_drive($vmid, $v);
2703                     } else {
2704                         warn "vm $vmid - unable to parse value of '$key'\n";
2705                         next;
2706                     }
2707                 }
2708
2709                 $conf->{$key} = $value;
2710             }
2711         }
2712     }
2713
2714     if (defined($descr)) {
2715         $descr =~ s/\s+$//;
2716         $conf->{description} = $descr;
2717     }
2718     delete $res->{snapstate}; # just to be sure
2719
2720     return $res;
2721 }
2722
2723 sub write_vm_config {
2724     my ($filename, $conf) = @_;
2725
2726     delete $conf->{snapstate}; # just to be sure
2727
2728     if ($conf->{cdrom}) {
2729         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2730         $conf->{ide2} = $conf->{cdrom};
2731         delete $conf->{cdrom};
2732     }
2733
2734     # we do not use 'smp' any longer
2735     if ($conf->{sockets}) {
2736         delete $conf->{smp};
2737     } elsif ($conf->{smp}) {
2738         $conf->{sockets} = $conf->{smp};
2739         delete $conf->{cores};
2740         delete $conf->{smp};
2741     }
2742
2743     my $used_volids = {};
2744
2745     my $cleanup_config = sub {
2746         my ($cref, $pending, $snapname) = @_;
2747
2748         foreach my $key (keys %$cref) {
2749             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2750                 $key eq 'snapstate' || $key eq 'pending';
2751             my $value = $cref->{$key};
2752             if ($key eq 'delete') {
2753                 die "propertry 'delete' is only allowed in [PENDING]\n"
2754                     if !$pending;
2755                 # fixme: check syntax?
2756                 next;
2757             }
2758             eval { $value = check_type($key, $value); };
2759             die "unable to parse value of '$key' - $@" if $@;
2760
2761             $cref->{$key} = $value;
2762
2763             if (!$snapname && is_valid_drivename($key)) {
2764                 my $drive = parse_drive($key, $value);
2765                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2766             }
2767         }
2768     };
2769
2770     &$cleanup_config($conf);
2771
2772     &$cleanup_config($conf->{pending}, 1);
2773
2774     foreach my $snapname (keys %{$conf->{snapshots}}) {
2775         die "internal error" if $snapname eq 'pending';
2776         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2777     }
2778
2779     # remove 'unusedX' settings if we re-add a volume
2780     foreach my $key (keys %$conf) {
2781         my $value = $conf->{$key};
2782         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2783             delete $conf->{$key};
2784         }
2785     }
2786
2787     my $generate_raw_config = sub {
2788         my ($conf, $pending) = @_;
2789
2790         my $raw = '';
2791
2792         # add description as comment to top of file
2793         if (defined(my $descr = $conf->{description})) {
2794             if ($descr) {
2795                 foreach my $cl (split(/\n/, $descr)) {
2796                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2797                 }
2798             } else {
2799                 $raw .= "#\n" if $pending;
2800             }
2801         }
2802
2803         foreach my $key (sort keys %$conf) {
2804             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2805             $raw .= "$key: $conf->{$key}\n";
2806         }
2807         return $raw;
2808     };
2809
2810     my $raw = &$generate_raw_config($conf);
2811
2812     if (scalar(keys %{$conf->{pending}})){
2813         $raw .= "\n[PENDING]\n";
2814         $raw .= &$generate_raw_config($conf->{pending}, 1);
2815     }
2816
2817     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2818         $raw .= "\n[$snapname]\n";
2819         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2820     }
2821
2822     return $raw;
2823 }
2824
2825 sub load_defaults {
2826
2827     my $res = {};
2828
2829     # we use static defaults from our JSON schema configuration
2830     foreach my $key (keys %$confdesc) {
2831         if (defined(my $default = $confdesc->{$key}->{default})) {
2832             $res->{$key} = $default;
2833         }
2834     }
2835
2836     return $res;
2837 }
2838
2839 sub config_list {
2840     my $vmlist = PVE::Cluster::get_vmlist();
2841     my $res = {};
2842     return $res if !$vmlist || !$vmlist->{ids};
2843     my $ids = $vmlist->{ids};
2844
2845     foreach my $vmid (keys %$ids) {
2846         my $d = $ids->{$vmid};
2847         next if !$d->{node} || $d->{node} ne $nodename;
2848         next if !$d->{type} || $d->{type} ne 'qemu';
2849         $res->{$vmid}->{exists} = 1;
2850     }
2851     return $res;
2852 }
2853
2854 # test if VM uses local resources (to prevent migration)
2855 sub check_local_resources {
2856     my ($conf, $noerr) = @_;
2857
2858     my @loc_res = ();
2859
2860     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2861     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2862
2863     push @loc_res, "ivshmem" if $conf->{ivshmem};
2864
2865     foreach my $k (keys %$conf) {
2866         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2867         # sockets are safe: they will recreated be on the target side post-migrate
2868         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2869         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2870     }
2871
2872     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2873
2874     return \@loc_res;
2875 }
2876
2877 # check if used storages are available on all nodes (use by migrate)
2878 sub check_storage_availability {
2879     my ($storecfg, $conf, $node) = @_;
2880
2881     foreach_drive($conf, sub {
2882         my ($ds, $drive) = @_;
2883
2884         my $volid = $drive->{file};
2885         return if !$volid;
2886
2887         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2888         return if !$sid;
2889
2890         # check if storage is available on both nodes
2891         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2892         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2893    });
2894 }
2895
2896 # list nodes where all VM images are available (used by has_feature API)
2897 sub shared_nodes {
2898     my ($conf, $storecfg) = @_;
2899
2900     my $nodelist = PVE::Cluster::get_nodelist();
2901     my $nodehash = { map { $_ => 1 } @$nodelist };
2902     my $nodename = PVE::INotify::nodename();
2903
2904     foreach_drive($conf, sub {
2905         my ($ds, $drive) = @_;
2906
2907         my $volid = $drive->{file};
2908         return if !$volid;
2909
2910         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2911         if ($storeid) {
2912             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2913             if ($scfg->{disable}) {
2914                 $nodehash = {};
2915             } elsif (my $avail = $scfg->{nodes}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if !$avail->{$node};
2918                 }
2919             } elsif (!$scfg->{shared}) {
2920                 foreach my $node (keys %$nodehash) {
2921                     delete $nodehash->{$node} if $node ne $nodename
2922                 }
2923             }
2924         }
2925     });
2926
2927     return $nodehash
2928 }
2929
2930 sub check_local_storage_availability {
2931     my ($conf, $storecfg) = @_;
2932
2933     my $nodelist = PVE::Cluster::get_nodelist();
2934     my $nodehash = { map { $_ => {} } @$nodelist };
2935
2936     foreach_drive($conf, sub {
2937         my ($ds, $drive) = @_;
2938
2939         my $volid = $drive->{file};
2940         return if !$volid;
2941
2942         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2943         if ($storeid) {
2944             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2945
2946             if ($scfg->{disable}) {
2947                 foreach my $node (keys %$nodehash) {
2948                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2949                 }
2950             } elsif (my $avail = $scfg->{nodes}) {
2951                 foreach my $node (keys %$nodehash) {
2952                     if (!$avail->{$node}) {
2953                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2954                     }
2955                 }
2956             }
2957         }
2958     });
2959
2960     foreach my $node (values %$nodehash) {
2961         if (my $unavail = $node->{unavailable_storages}) {
2962             $node->{unavailable_storages} = [ sort keys %$unavail ];
2963         }
2964     }
2965
2966     return $nodehash
2967 }
2968
2969 sub check_cmdline {
2970     my ($pidfile, $pid) = @_;
2971
2972     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2973     if (defined($fh)) {
2974         my $line = <$fh>;
2975         $fh->close;
2976         return undef if !$line;
2977         my @param = split(/\0/, $line);
2978
2979         my $cmd = $param[0];
2980         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2981
2982         for (my $i = 0; $i < scalar (@param); $i++) {
2983             my $p = $param[$i];
2984             next if !$p;
2985             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2986                 my $p = $param[$i+1];
2987                 return 1 if $p && ($p eq $pidfile);
2988                 return undef;
2989             }
2990         }
2991     }
2992     return undef;
2993 }
2994
2995 sub check_running {
2996     my ($vmid, $nocheck, $node) = @_;
2997
2998     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2999
3000     die "unable to find configuration file for VM $vmid - no such machine\n"
3001         if !$nocheck && ! -f $filename;
3002
3003     my $pidfile = pidfile_name($vmid);
3004
3005     if (my $fd = IO::File->new("<$pidfile")) {
3006         my $st = stat($fd);
3007         my $line = <$fd>;
3008         close($fd);
3009
3010         my $mtime = $st->mtime;
3011         if ($mtime > time()) {
3012             warn "file '$filename' modified in future\n";
3013         }
3014
3015         if ($line =~ m/^(\d+)$/) {
3016             my $pid = $1;
3017             if (check_cmdline($pidfile, $pid)) {
3018                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3019                     return $pid;
3020                 }
3021             }
3022         }
3023     }
3024
3025     return undef;
3026 }
3027
3028 sub vzlist {
3029
3030     my $vzlist = config_list();
3031
3032     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3033
3034     while (defined(my $de = $fd->read)) {
3035         next if $de !~ m/^(\d+)\.pid$/;
3036         my $vmid = $1;
3037         next if !defined($vzlist->{$vmid});
3038         if (my $pid = check_running($vmid)) {
3039             $vzlist->{$vmid}->{pid} = $pid;
3040         }
3041     }
3042
3043     return $vzlist;
3044 }
3045
3046 sub disksize {
3047     my ($storecfg, $conf) = @_;
3048
3049     my $bootdisk = $conf->{bootdisk};
3050     return undef if !$bootdisk;
3051     return undef if !is_valid_drivename($bootdisk);
3052
3053     return undef if !$conf->{$bootdisk};
3054
3055     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3056     return undef if !defined($drive);
3057
3058     return undef if drive_is_cdrom($drive);
3059
3060     my $volid = $drive->{file};
3061     return undef if !$volid;
3062
3063     return $drive->{size};
3064 }
3065
3066 our $vmstatus_return_properties = {
3067     vmid => get_standard_option('pve-vmid'),
3068     status => {
3069         description => "Qemu process status.",
3070         type => 'string',
3071         enum => ['stopped', 'running'],
3072     },
3073     maxmem => {
3074         description => "Maximum memory in bytes.",
3075         type => 'integer',
3076         optional => 1,
3077         renderer => 'bytes',
3078     },
3079     maxdisk => {
3080         description => "Root disk size in bytes.",
3081         type => 'integer',
3082         optional => 1,
3083         renderer => 'bytes',
3084     },
3085     name => {
3086         description => "VM name.",
3087         type => 'string',
3088         optional => 1,
3089     },
3090     qmpstatus => {
3091         description => "Qemu QMP agent status.",
3092         type => 'string',
3093         optional => 1,
3094     },
3095     pid => {
3096         description => "PID of running qemu process.",
3097         type => 'integer',
3098         optional => 1,
3099     },
3100     uptime => {
3101         description => "Uptime.",
3102         type => 'integer',
3103         optional => 1,
3104         renderer => 'duration',
3105     },
3106     cpus => {
3107         description => "Maximum usable CPUs.",
3108         type => 'number',
3109         optional => 1,
3110     },
3111     lock => {
3112         description => "The current config lock, if any.",
3113         type => 'string',
3114         optional => 1,
3115     }
3116 };
3117
3118 my $last_proc_pid_stat;
3119
3120 # get VM status information
3121 # This must be fast and should not block ($full == false)
3122 # We only query KVM using QMP if $full == true (this can be slow)
3123 sub vmstatus {
3124     my ($opt_vmid, $full) = @_;
3125
3126     my $res = {};
3127
3128     my $storecfg = PVE::Storage::config();
3129
3130     my $list = vzlist();
3131     my $defaults = load_defaults();
3132
3133     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3134
3135     my $cpucount = $cpuinfo->{cpus} || 1;
3136
3137     foreach my $vmid (keys %$list) {
3138         next if $opt_vmid && ($vmid ne $opt_vmid);
3139
3140         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3141         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3142
3143         my $d = { vmid => $vmid };
3144         $d->{pid} = $list->{$vmid}->{pid};
3145
3146         # fixme: better status?
3147         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3148
3149         my $size = disksize($storecfg, $conf);
3150         if (defined($size)) {
3151             $d->{disk} = 0; # no info available
3152             $d->{maxdisk} = $size;
3153         } else {
3154             $d->{disk} = 0;
3155             $d->{maxdisk} = 0;
3156         }
3157
3158         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3159             * ($conf->{cores} || $defaults->{cores});
3160         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3161         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3162
3163         $d->{name} = $conf->{name} || "VM $vmid";
3164         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3165             : $defaults->{memory}*(1024*1024);
3166
3167         if ($conf->{balloon}) {
3168             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3169             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3170                 : $defaults->{shares};
3171         }
3172
3173         $d->{uptime} = 0;
3174         $d->{cpu} = 0;
3175         $d->{mem} = 0;
3176
3177         $d->{netout} = 0;
3178         $d->{netin} = 0;
3179
3180         $d->{diskread} = 0;
3181         $d->{diskwrite} = 0;
3182
3183         $d->{template} = PVE::QemuConfig->is_template($conf);
3184
3185         $d->{serial} = 1 if conf_has_serial($conf);
3186         $d->{lock} = $conf->{lock} if $conf->{lock};
3187
3188         $res->{$vmid} = $d;
3189     }
3190
3191     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3192     foreach my $dev (keys %$netdev) {
3193         next if $dev !~ m/^tap([1-9]\d*)i/;
3194         my $vmid = $1;
3195         my $d = $res->{$vmid};
3196         next if !$d;
3197
3198         $d->{netout} += $netdev->{$dev}->{receive};
3199         $d->{netin} += $netdev->{$dev}->{transmit};
3200
3201         if ($full) {
3202             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3203             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3204         }
3205
3206     }
3207
3208     my $ctime = gettimeofday;
3209
3210     foreach my $vmid (keys %$list) {
3211
3212         my $d = $res->{$vmid};
3213         my $pid = $d->{pid};
3214         next if !$pid;
3215
3216         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3217         next if !$pstat; # not running
3218
3219         my $used = $pstat->{utime} + $pstat->{stime};
3220
3221         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3222
3223         if ($pstat->{vsize}) {
3224             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3225         }
3226
3227         my $old = $last_proc_pid_stat->{$pid};
3228         if (!$old) {
3229             $last_proc_pid_stat->{$pid} = {
3230                 time => $ctime,
3231                 used => $used,
3232                 cpu => 0,
3233             };
3234             next;
3235         }
3236
3237         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3238
3239         if ($dtime > 1000) {
3240             my $dutime = $used -  $old->{used};
3241
3242             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3243             $last_proc_pid_stat->{$pid} = {
3244                 time => $ctime,
3245                 used => $used,
3246                 cpu => $d->{cpu},
3247             };
3248         } else {
3249             $d->{cpu} = $old->{cpu};
3250         }
3251     }
3252
3253     return $res if !$full;
3254
3255     my $qmpclient = PVE::QMPClient->new();
3256
3257     my $ballooncb = sub {
3258         my ($vmid, $resp) = @_;
3259
3260         my $info = $resp->{'return'};
3261         return if !$info->{max_mem};
3262
3263         my $d = $res->{$vmid};
3264
3265         # use memory assigned to VM
3266         $d->{maxmem} = $info->{max_mem};
3267         $d->{balloon} = $info->{actual};
3268
3269         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3270             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3271             $d->{freemem} = $info->{free_mem};
3272         }
3273
3274         $d->{ballooninfo} = $info;
3275     };
3276
3277     my $blockstatscb = sub {
3278         my ($vmid, $resp) = @_;
3279         my $data = $resp->{'return'} || [];
3280         my $totalrdbytes = 0;
3281         my $totalwrbytes = 0;
3282
3283         for my $blockstat (@$data) {
3284             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3285             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3286
3287             $blockstat->{device} =~ s/drive-//;
3288             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3289         }
3290         $res->{$vmid}->{diskread} = $totalrdbytes;
3291         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3292     };
3293
3294     my $statuscb = sub {
3295         my ($vmid, $resp) = @_;
3296
3297         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3298         # this fails if ballon driver is not loaded, so this must be
3299         # the last commnand (following command are aborted if this fails).
3300         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3301
3302         my $status = 'unknown';
3303         if (!defined($status = $resp->{'return'}->{status})) {
3304             warn "unable to get VM status\n";
3305             return;
3306         }
3307
3308         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3309     };
3310
3311     foreach my $vmid (keys %$list) {
3312         next if $opt_vmid && ($vmid ne $opt_vmid);
3313         next if !$res->{$vmid}->{pid}; # not running
3314         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3315     }
3316
3317     $qmpclient->queue_execute(undef, 2);
3318
3319     foreach my $vmid (keys %$list) {
3320         next if $opt_vmid && ($vmid ne $opt_vmid);
3321         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3322     }
3323
3324     return $res;
3325 }
3326
3327 sub foreach_drive {
3328     my ($conf, $func, @param) = @_;
3329
3330     foreach my $ds (valid_drive_names()) {
3331         next if !defined($conf->{$ds});
3332
3333         my $drive = parse_drive($ds, $conf->{$ds});
3334         next if !$drive;
3335
3336         &$func($ds, $drive, @param);
3337     }
3338 }
3339
3340 sub foreach_volid {
3341     my ($conf, $func, @param) = @_;
3342
3343     my $volhash = {};
3344
3345     my $test_volid = sub {
3346         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3347
3348         return if !$volid;
3349
3350         $volhash->{$volid}->{cdrom} //= 1;
3351         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3352
3353         $volhash->{$volid}->{replicate} //= 0;
3354         $volhash->{$volid}->{replicate} = 1 if $replicate;
3355
3356         $volhash->{$volid}->{shared} //= 0;
3357         $volhash->{$volid}->{shared} = 1 if $shared;
3358
3359         $volhash->{$volid}->{referenced_in_config} //= 0;
3360         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3361
3362         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3363             if defined($snapname);
3364         $volhash->{$volid}->{size} = $size if $size;
3365     };
3366
3367     foreach_drive($conf, sub {
3368         my ($ds, $drive) = @_;
3369         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3370     });
3371
3372     foreach my $snapname (keys %{$conf->{snapshots}}) {
3373         my $snap = $conf->{snapshots}->{$snapname};
3374         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3375         foreach_drive($snap, sub {
3376             my ($ds, $drive) = @_;
3377             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3378         });
3379     }
3380
3381     foreach my $volid (keys %$volhash) {
3382         &$func($volid, $volhash->{$volid}, @param);
3383     }
3384 }
3385
3386 sub conf_has_serial {
3387     my ($conf) = @_;
3388
3389     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3390         if ($conf->{"serial$i"}) {
3391             return 1;
3392         }
3393     }
3394
3395     return 0;
3396 }
3397
3398 sub conf_has_audio {
3399     my ($conf, $id) = @_;
3400
3401     $id //= 0;
3402     my $audio = $conf->{"audio$id"};
3403     return undef if !defined($audio);
3404
3405     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3406     my $audiodriver = $audioproperties->{driver} // 'spice';
3407
3408     return {
3409         dev => $audioproperties->{device},
3410         dev_id => "audiodev$id",
3411         backend => $audiodriver,
3412         backend_id => "$audiodriver-backend${id}",
3413     };
3414 }
3415
3416 sub vga_conf_has_spice {
3417     my ($vga) = @_;
3418
3419     my $vgaconf = parse_vga($vga);
3420     my $vgatype = $vgaconf->{type};
3421     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3422
3423     return $1 || 1;
3424 }
3425
3426 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3427 sub get_host_arch() {
3428     $host_arch = (POSIX::uname())[4] if !$host_arch;
3429     return $host_arch;
3430 }
3431
3432 sub is_native($) {
3433     my ($arch) = @_;
3434     return get_host_arch() eq $arch;
3435 }
3436
3437 my $default_machines = {
3438     x86_64 => 'pc',
3439     aarch64 => 'virt',
3440 };
3441
3442 sub get_basic_machine_info {
3443     my ($conf, $forcemachine) = @_;
3444
3445     my $arch = $conf->{arch} // get_host_arch();
3446     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3447     return ($arch, $machine);
3448 }
3449
3450 sub get_ovmf_files($) {
3451     my ($arch) = @_;
3452
3453     my $ovmf = $OVMF->{$arch}
3454         or die "no OVMF images known for architecture '$arch'\n";
3455
3456     return @$ovmf;
3457 }
3458
3459 my $Arch2Qemu = {
3460     aarch64 => '/usr/bin/qemu-system-aarch64',
3461     x86_64 => '/usr/bin/qemu-system-x86_64',
3462 };
3463 sub get_command_for_arch($) {
3464     my ($arch) = @_;
3465     return '/usr/bin/kvm' if is_native($arch);
3466
3467     my $cmd = $Arch2Qemu->{$arch}
3468         or die "don't know how to emulate architecture '$arch'\n";
3469     return $cmd;
3470 }
3471
3472 sub get_cpu_options {
3473     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3474
3475     my $cpuFlags = [];
3476     my $ostype = $conf->{ostype};
3477
3478     my $cpu = $kvm ? "kvm64" : "qemu64";
3479     if ($arch eq 'aarch64') {
3480         $cpu = 'cortex-a57';
3481     }
3482     my $hv_vendor_id;
3483     if (my $cputype = $conf->{cpu}) {
3484         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3485             or die "Cannot parse cpu description: $cputype\n";
3486         $cpu = $cpuconf->{cputype};
3487         $kvm_off = 1 if $cpuconf->{hidden};
3488         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3489
3490         if (defined(my $flags = $cpuconf->{flags})) {
3491             push @$cpuFlags, split(";", $flags);
3492         }
3493     }
3494
3495     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3496
3497     push @$cpuFlags , '-x2apic'
3498         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3499
3500     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3501
3502     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3503
3504     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3505
3506         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3507         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3508     }
3509
3510     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3511
3512     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3513
3514     push @$cpuFlags, 'kvm=off' if $kvm_off;
3515
3516     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3517         push @$cpuFlags, "vendor=${cpu_vendor}"
3518             if $cpu_vendor ne 'default';
3519     } elsif ($arch ne 'aarch64') {
3520         die "internal error"; # should not happen
3521     }
3522
3523     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3524
3525     return ('-cpu', $cpu);
3526 }
3527
3528 sub config_to_command {
3529     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3530
3531     my $cmd = [];
3532     my $globalFlags = [];
3533     my $machineFlags = [];
3534     my $rtcFlags = [];
3535     my $devices = [];
3536     my $pciaddr = '';
3537     my $bridges = {};
3538     my $vernum = 0; # unknown
3539     my $ostype = $conf->{ostype};
3540     my $winversion = windows_version($ostype);
3541     my $kvm = $conf->{kvm};
3542
3543     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3544     my $kvm_binary = get_command_for_arch($arch);
3545     my $kvmver = kvm_user_version($kvm_binary);
3546     $kvm //= 1 if is_native($arch);
3547
3548     if ($kvm) {
3549         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3550             if !defined kvm_version();
3551     }
3552
3553     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3554         $vernum = $1*1000000+$2*1000;
3555     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3556         $vernum = $1*1000000+$2*1000+$3;
3557     }
3558
3559     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3560
3561     my $q35 = machine_type_is_q35($conf);
3562     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3563     my $use_old_bios_files = undef;
3564     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3565
3566     my $cpuunits = defined($conf->{cpuunits}) ?
3567             $conf->{cpuunits} : $defaults->{cpuunits};
3568
3569     push @$cmd, $kvm_binary;
3570
3571     push @$cmd, '-id', $vmid;
3572
3573     my $vmname = $conf->{name} || "vm$vmid";
3574
3575     push @$cmd, '-name', $vmname;
3576
3577     my $use_virtio = 0;
3578
3579     my $qmpsocket = qmp_socket($vmid);
3580     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3581     push @$cmd, '-mon', "chardev=qmp,mode=control";
3582
3583     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3584         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3585         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3586     }
3587
3588     push @$cmd, '-pidfile' , pidfile_name($vmid);
3589
3590     push @$cmd, '-daemonize';
3591
3592     if ($conf->{smbios1}) {
3593         my $smbios_conf = parse_smbios1($conf->{smbios1});
3594         if ($smbios_conf->{base64}) {
3595             # Do not pass base64 flag to qemu
3596             delete $smbios_conf->{base64};
3597             my $smbios_string = "";
3598             foreach my $key (keys %$smbios_conf) {
3599                 my $value;
3600                 if ($key eq "uuid") {
3601                     $value = $smbios_conf->{uuid}
3602                 } else {
3603                     $value = decode_base64($smbios_conf->{$key});
3604                 }
3605                 # qemu accepts any binary data, only commas need escaping by double comma
3606                 $value =~ s/,/,,/g;
3607                 $smbios_string .= "," . $key . "=" . $value if $value;
3608             }
3609             push @$cmd, '-smbios', "type=1" . $smbios_string;
3610         } else {
3611             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3612         }
3613     }
3614
3615     if ($conf->{vmgenid}) {
3616         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3617     }
3618
3619     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3620     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3621         die "uefi base image not found\n" if ! -f $ovmf_code;
3622
3623         my $path;
3624         my $format;
3625         if (my $efidisk = $conf->{efidisk0}) {
3626             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3627             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3628             $format = $d->{format};
3629             if ($storeid) {
3630                 $path = PVE::Storage::path($storecfg, $d->{file});
3631                 if (!defined($format)) {
3632                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3633                     $format = qemu_img_format($scfg, $volname);
3634                 }
3635             } else {
3636                 $path = $d->{file};
3637                 die "efidisk format must be specified\n"
3638                     if !defined($format);
3639             }
3640         } else {
3641             warn "no efidisk configured! Using temporary efivars disk.\n";
3642             $path = "/tmp/$vmid-ovmf.fd";
3643             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3644             $format = 'raw';
3645         }
3646
3647         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3648         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3649     }
3650
3651     # load q35 config
3652     if ($q35) {
3653         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3654         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3655             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3656         } else {
3657             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3658         }
3659     }
3660
3661     # add usb controllers
3662     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3663     push @$devices, @usbcontrollers if @usbcontrollers;
3664     my $vga = parse_vga($conf->{vga});
3665
3666     my $qxlnum = vga_conf_has_spice($conf->{vga});
3667     $vga->{type} = 'qxl' if $qxlnum;
3668
3669     if (!$vga->{type}) {
3670         if ($arch eq 'aarch64') {
3671             $vga->{type} = 'virtio';
3672         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3673             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3674         } else {
3675             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3676         }
3677     }
3678
3679     # enable absolute mouse coordinates (needed by vnc)
3680     my $tablet;
3681     if (defined($conf->{tablet})) {
3682         $tablet = $conf->{tablet};
3683     } else {
3684         $tablet = $defaults->{tablet};
3685         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3686         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3687     }
3688
3689     if ($tablet) {
3690         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3691         my $kbd = print_keyboarddevice_full($conf, $arch);
3692         push @$devices, '-device', $kbd if defined($kbd);
3693     }
3694
3695     my $kvm_off = 0;
3696     my $gpu_passthrough;
3697
3698     # host pci devices
3699     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3700         my $id = "hostpci$i";
3701         my $d = parse_hostpci($conf->{$id});
3702         next if !$d;
3703
3704         if (my $pcie = $d->{pcie}) {
3705             die "q35 machine model is not enabled" if !$q35;
3706             # win7 wants to have the pcie devices directly on the pcie bus
3707             # instead of in the root port
3708             if ($winversion == 7) {
3709                 $pciaddr = print_pcie_addr("${id}bus0");
3710             } else {
3711                 # add more root ports if needed, 4 are present by default
3712                 # by pve-q35 cfgs, rest added here on demand.
3713                 if ($i > 3) {
3714                     push @$devices, '-device', print_pcie_root_port($i);
3715                 }
3716                 $pciaddr = print_pcie_addr($id);
3717             }
3718         } else {
3719             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3720         }
3721
3722         my $xvga = '';
3723         if ($d->{'x-vga'}) {
3724             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3725             $kvm_off = 1;
3726             $vga->{type} = 'none' if !defined($conf->{vga});
3727             $gpu_passthrough = 1;
3728         }
3729
3730         my $pcidevices = $d->{pciid};
3731         my $multifunction = 1 if @$pcidevices > 1;
3732
3733         my $sysfspath;
3734         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3735             my $pci_id = $pcidevices->[0]->{id};
3736             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3737             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3738         } elsif ($d->{mdev}) {
3739             warn "ignoring mediated device '$id' with multifunction device\n";
3740         }
3741
3742         my $j=0;
3743         foreach my $pcidevice (@$pcidevices) {
3744             my $devicestr = "vfio-pci";
3745
3746             if ($sysfspath) {
3747                 $devicestr .= ",sysfsdev=$sysfspath";
3748             } else {
3749                 $devicestr .= ",host=$pcidevice->{id}";
3750             }
3751
3752             my $mf_addr = $multifunction ? ".$j" : '';
3753             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3754
3755             if ($j == 0) {
3756                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3757                 $devicestr .= "$xvga";
3758                 $devicestr .= ",multifunction=on" if $multifunction;
3759                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3760             }
3761
3762             push @$devices, '-device', $devicestr;
3763             $j++;
3764         }
3765     }
3766
3767     # usb devices
3768     my $usb_dev_features = {};
3769     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3770
3771     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3772     push @$devices, @usbdevices if @usbdevices;
3773     # serial devices
3774     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3775         if (my $path = $conf->{"serial$i"}) {
3776             if ($path eq 'socket') {
3777                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3778                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3779                 # On aarch64, serial0 is the UART device. Qemu only allows
3780                 # connecting UART devices via the '-serial' command line, as
3781                 # the device has a fixed slot on the hardware...
3782                 if ($arch eq 'aarch64' && $i == 0) {
3783                     push @$devices, '-serial', "chardev:serial$i";
3784                 } else {
3785                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3786                 }
3787             } else {
3788                 die "no such serial device\n" if ! -c $path;
3789                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3790                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3791             }
3792         }
3793     }
3794
3795     # parallel devices
3796     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3797         if (my $path = $conf->{"parallel$i"}) {
3798             die "no such parallel device\n" if ! -c $path;
3799             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3800             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3801             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3802         }
3803     }
3804
3805     if (my $audio = conf_has_audio($conf)) {
3806
3807         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3808
3809         my $id = $audio->{dev_id};
3810         if ($audio->{dev} eq 'AC97') {
3811             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3812         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3813             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3814             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3815             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3816         } else {
3817             die "unkown audio device '$audio->{dev}', implement me!";
3818         }
3819
3820         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3821     }
3822
3823     my $sockets = 1;
3824     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3825     $sockets = $conf->{sockets} if  $conf->{sockets};
3826
3827     my $cores = $conf->{cores} || 1;
3828
3829     my $maxcpus = $sockets * $cores;
3830
3831     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3832
3833     my $allowed_vcpus = $cpuinfo->{cpus};
3834
3835     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3836         if ($allowed_vcpus < $maxcpus);
3837
3838     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3839
3840         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3841         for (my $i = 2; $i <= $vcpus; $i++)  {
3842             my $cpustr = print_cpu_device($conf,$i);
3843             push @$cmd, '-device', $cpustr;
3844         }
3845
3846     } else {
3847
3848         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3849     }
3850     push @$cmd, '-nodefaults';
3851
3852     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3853
3854     my $bootindex_hash = {};
3855     my $i = 1;
3856     foreach my $o (split(//, $bootorder)) {
3857         $bootindex_hash->{$o} = $i*100;
3858         $i++;
3859     }
3860
3861     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3862
3863     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3864
3865     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3866
3867     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3868         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3869         my $socket = vnc_socket($vmid);
3870         push @$cmd,  '-vnc', "unix:$socket,password";
3871     } else {
3872         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3873         push @$cmd, '-nographic';
3874     }
3875
3876     # time drift fix
3877     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3878
3879     my $useLocaltime = $conf->{localtime};
3880
3881     if ($winversion >= 5) { # windows
3882         $useLocaltime = 1 if !defined($conf->{localtime});
3883
3884         # use time drift fix when acpi is enabled
3885         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3886             $tdf = 1 if !defined($conf->{tdf});
3887         }
3888     }
3889
3890     if ($winversion >= 6) {
3891         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3892         push @$cmd, '-no-hpet';
3893     }
3894
3895     push @$rtcFlags, 'driftfix=slew' if $tdf;
3896
3897     if (!$kvm) {
3898         push @$machineFlags, 'accel=tcg';
3899     }
3900
3901     if ($machine_type) {
3902         push @$machineFlags, "type=${machine_type}";
3903     }
3904
3905     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3906         push @$rtcFlags, "base=$conf->{startdate}";
3907     } elsif ($useLocaltime) {
3908         push @$rtcFlags, 'base=localtime';
3909     }
3910
3911     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3912
3913     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3914
3915     push @$cmd, '-S' if $conf->{freeze};
3916
3917     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3918
3919     if (parse_guest_agent($conf)->{enabled}) {
3920         my $qgasocket = qmp_socket($vmid, 1);
3921         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3922         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3923         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3924         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3925     }
3926
3927     my $spice_port;
3928
3929     if ($qxlnum) {
3930         if ($qxlnum > 1) {
3931             if ($winversion){
3932                 for(my $i = 1; $i < $qxlnum; $i++){
3933                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3934                 }
3935             } else {
3936                 # assume other OS works like Linux
3937                 my ($ram, $vram) = ("134217728", "67108864");
3938                 if ($vga->{memory}) {
3939                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3940                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3941                 }
3942                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3943                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3944             }
3945         }
3946
3947         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3948
3949         my $nodename = PVE::INotify::nodename();
3950         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3951         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3952         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3953
3954         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3955         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3956         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3957
3958         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3959         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3960
3961         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3962         if ($spice_enhancement->{foldersharing}) {
3963             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3964             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3965         }
3966
3967         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3968         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3969         push @$devices, '-spice', "$spice_opts";
3970     }
3971
3972     # enable balloon by default, unless explicitly disabled
3973     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3974         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3975         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3976     }
3977
3978     if ($conf->{watchdog}) {
3979         my $wdopts = parse_watchdog($conf->{watchdog});
3980         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3981         my $watchdog = $wdopts->{model} || 'i6300esb';
3982         push @$devices, '-device', "$watchdog$pciaddr";
3983         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3984     }
3985
3986     my $vollist = [];
3987     my $scsicontroller = {};
3988     my $ahcicontroller = {};
3989     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3990
3991     # Add iscsi initiator name if available
3992     if (my $initiator = get_initiator_name()) {
3993         push @$devices, '-iscsi', "initiator-name=$initiator";
3994     }
3995
3996     foreach_drive($conf, sub {
3997         my ($ds, $drive) = @_;
3998
3999         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4000             push @$vollist, $drive->{file};
4001         }
4002
4003         # ignore efidisk here, already added in bios/fw handling code above
4004         return if $drive->{interface} eq 'efidisk';
4005
4006         $use_virtio = 1 if $ds =~ m/^virtio/;
4007
4008         if (drive_is_cdrom ($drive)) {
4009             if ($bootindex_hash->{d}) {
4010                 $drive->{bootindex} = $bootindex_hash->{d};
4011                 $bootindex_hash->{d} += 1;
4012             }
4013         } else {
4014             if ($bootindex_hash->{c}) {
4015                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4016                 $bootindex_hash->{c} += 1;
4017             }
4018         }
4019
4020         if($drive->{interface} eq 'virtio'){
4021            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4022         }
4023
4024         if ($drive->{interface} eq 'scsi') {
4025
4026             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4027
4028             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4029             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4030
4031             my $iothread = '';
4032             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4033                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4034                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4035             } elsif ($drive->{iothread}) {
4036                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4037             }
4038
4039             my $queues = '';
4040             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4041                 $queues = ",num_queues=$drive->{queues}";
4042             }
4043
4044             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4045             $scsicontroller->{$controller}=1;
4046         }
4047
4048         if ($drive->{interface} eq 'sata') {
4049            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4050            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4051            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4052            $ahcicontroller->{$controller}=1;
4053         }
4054
4055         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4056         push @$devices, '-drive',$drive_cmd;
4057         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4058     });
4059
4060     for (my $i = 0; $i < $MAX_NETS; $i++) {
4061          next if !$conf->{"net$i"};
4062          my $d = parse_net($conf->{"net$i"});
4063          next if !$d;
4064
4065          $use_virtio = 1 if $d->{model} eq 'virtio';
4066
4067          if ($bootindex_hash->{n}) {
4068             $d->{bootindex} = $bootindex_hash->{n};
4069             $bootindex_hash->{n} += 1;
4070          }
4071
4072          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4073          push @$devices, '-netdev', $netdevfull;
4074
4075          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4076          push @$devices, '-device', $netdevicefull;
4077     }
4078
4079     if ($conf->{ivshmem}) {
4080         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4081
4082         my $bus;
4083         if ($q35) {
4084             $bus = print_pcie_addr("ivshmem");
4085         } else {
4086             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4087         }
4088
4089         my $ivshmem_name = $ivshmem->{name} // $vmid;
4090         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4091
4092         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4093         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4094     }
4095
4096     if (!$q35) {
4097         # add pci bridges
4098         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4099            $bridges->{1} = 1;
4100            $bridges->{2} = 1;
4101         }
4102
4103         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4104
4105         for my $k (sort {$b cmp $a} keys %$bridges) {
4106             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4107             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4108         }
4109     }
4110
4111     push @$cmd, @$devices;
4112     push @$cmd, '-rtc', join(',', @$rtcFlags)
4113         if scalar(@$rtcFlags);
4114     push @$cmd, '-machine', join(',', @$machineFlags)
4115         if scalar(@$machineFlags);
4116     push @$cmd, '-global', join(',', @$globalFlags)
4117         if scalar(@$globalFlags);
4118
4119     if (my $vmstate = $conf->{vmstate}) {
4120         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4121         push @$vollist, $vmstate;
4122         push @$cmd, '-loadstate', $statepath;
4123     }
4124
4125     # add custom args
4126     if ($conf->{args}) {
4127         my $aa = PVE::Tools::split_args($conf->{args});
4128         push @$cmd, @$aa;
4129     }
4130
4131     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4132 }
4133
4134 sub vnc_socket {
4135     my ($vmid) = @_;
4136     return "${var_run_tmpdir}/$vmid.vnc";
4137 }
4138
4139 sub spice_port {
4140     my ($vmid) = @_;
4141
4142     my $res = vm_mon_cmd($vmid, 'query-spice');
4143
4144     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4145 }
4146
4147 sub qmp_socket {
4148     my ($vmid, $qga, $name) = @_;
4149     my $sockettype = $qga ? 'qga' : 'qmp';
4150     my $ext = $name ? '-'.$name : '';
4151     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4152 }
4153
4154 sub pidfile_name {
4155     my ($vmid) = @_;
4156     return "${var_run_tmpdir}/$vmid.pid";
4157 }
4158
4159 sub vm_devices_list {
4160     my ($vmid) = @_;
4161
4162     my $res = vm_mon_cmd($vmid, 'query-pci');
4163     my $devices_to_check = [];
4164     my $devices = {};
4165     foreach my $pcibus (@$res) {
4166         push @$devices_to_check, @{$pcibus->{devices}},
4167     }
4168
4169     while (@$devices_to_check) {
4170         my $to_check = [];
4171         for my $d (@$devices_to_check) {
4172             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4173             next if !$d->{'pci_bridge'};
4174
4175             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4176             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4177         }
4178         $devices_to_check = $to_check;
4179     }
4180
4181     my $resblock = vm_mon_cmd($vmid, 'query-block');
4182     foreach my $block (@$resblock) {
4183         if($block->{device} =~ m/^drive-(\S+)/){
4184                 $devices->{$1} = 1;
4185         }
4186     }
4187
4188     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4189     foreach my $mice (@$resmice) {
4190         if ($mice->{name} eq 'QEMU HID Tablet') {
4191             $devices->{tablet} = 1;
4192             last;
4193         }
4194     }
4195
4196     # for usb devices there is no query-usb
4197     # but we can iterate over the entries in
4198     # qom-list path=/machine/peripheral
4199     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4200     foreach my $per (@$resperipheral) {
4201         if ($per->{name} =~ m/^usb\d+$/) {
4202             $devices->{$per->{name}} = 1;
4203         }
4204     }
4205
4206     return $devices;
4207 }
4208
4209 sub vm_deviceplug {
4210     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4211
4212     my $q35 = machine_type_is_q35($conf);
4213
4214     my $devices_list = vm_devices_list($vmid);
4215     return 1 if defined($devices_list->{$deviceid});
4216
4217     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4218
4219     if ($deviceid eq 'tablet') {
4220
4221         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4222
4223     } elsif ($deviceid eq 'keyboard') {
4224
4225         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4226
4227     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4228
4229         die "usb hotplug currently not reliable\n";
4230         # since we can't reliably hot unplug all added usb devices
4231         # and usb passthrough disables live migration
4232         # we disable usb hotplugging for now
4233         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4234
4235     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4236
4237         qemu_iothread_add($vmid, $deviceid, $device);
4238
4239         qemu_driveadd($storecfg, $vmid, $device);
4240         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4241
4242         qemu_deviceadd($vmid, $devicefull);
4243         eval { qemu_deviceaddverify($vmid, $deviceid); };
4244         if (my $err = $@) {
4245             eval { qemu_drivedel($vmid, $deviceid); };
4246             warn $@ if $@;
4247             die $err;
4248         }
4249
4250     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4251
4252
4253         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4254         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4255         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4256
4257         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4258
4259         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4260             qemu_iothread_add($vmid, $deviceid, $device);
4261             $devicefull .= ",iothread=iothread-$deviceid";
4262         }
4263
4264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4265             $devicefull .= ",num_queues=$device->{queues}";
4266         }
4267
4268         qemu_deviceadd($vmid, $devicefull);
4269         qemu_deviceaddverify($vmid, $deviceid);
4270
4271     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4272
4273         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4274         qemu_driveadd($storecfg, $vmid, $device);
4275
4276         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4277         eval { qemu_deviceadd($vmid, $devicefull); };
4278         if (my $err = $@) {
4279             eval { qemu_drivedel($vmid, $deviceid); };
4280             warn $@ if $@;
4281             die $err;
4282         }
4283
4284     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4285
4286         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4287
4288         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4289         my $use_old_bios_files = undef;
4290         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4291
4292         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4293         qemu_deviceadd($vmid, $netdevicefull);
4294         eval {
4295             qemu_deviceaddverify($vmid, $deviceid);
4296             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4297         };
4298         if (my $err = $@) {
4299             eval { qemu_netdevdel($vmid, $deviceid); };
4300             warn $@ if $@;
4301             die $err;
4302         }
4303
4304     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4305
4306         my $bridgeid = $2;
4307         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4308         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4309
4310         qemu_deviceadd($vmid, $devicefull);
4311         qemu_deviceaddverify($vmid, $deviceid);
4312
4313     } else {
4314         die "can't hotplug device '$deviceid'\n";
4315     }
4316
4317     return 1;
4318 }
4319
4320 # fixme: this should raise exceptions on error!
4321 sub vm_deviceunplug {
4322     my ($vmid, $conf, $deviceid) = @_;
4323
4324     my $devices_list = vm_devices_list($vmid);
4325     return 1 if !defined($devices_list->{$deviceid});
4326
4327     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4328
4329     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4330
4331         qemu_devicedel($vmid, $deviceid);
4332
4333     } elsif ($deviceid =~ m/^usb\d+$/) {
4334
4335         die "usb hotplug currently not reliable\n";
4336         # when unplugging usb devices this way,
4337         # there may be remaining usb controllers/hubs
4338         # so we disable it for now
4339         qemu_devicedel($vmid, $deviceid);
4340         qemu_devicedelverify($vmid, $deviceid);
4341
4342     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4343
4344         qemu_devicedel($vmid, $deviceid);
4345         qemu_devicedelverify($vmid, $deviceid);
4346         qemu_drivedel($vmid, $deviceid);
4347         qemu_iothread_del($conf, $vmid, $deviceid);
4348
4349     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4350
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_devicedelverify($vmid, $deviceid);
4353         qemu_iothread_del($conf, $vmid, $deviceid);
4354
4355     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4356
4357         qemu_devicedel($vmid, $deviceid);
4358         qemu_drivedel($vmid, $deviceid);
4359         qemu_deletescsihw($conf, $vmid, $deviceid);
4360
4361     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4362
4363         qemu_devicedel($vmid, $deviceid);
4364         qemu_devicedelverify($vmid, $deviceid);
4365         qemu_netdevdel($vmid, $deviceid);
4366
4367     } else {
4368         die "can't unplug device '$deviceid'\n";
4369     }
4370
4371     return 1;
4372 }
4373
4374 sub qemu_deviceadd {
4375     my ($vmid, $devicefull) = @_;
4376
4377     $devicefull = "driver=".$devicefull;
4378     my %options =  split(/[=,]/, $devicefull);
4379
4380     vm_mon_cmd($vmid, "device_add" , %options);
4381 }
4382
4383 sub qemu_devicedel {
4384     my ($vmid, $deviceid) = @_;
4385
4386     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4387 }
4388
4389 sub qemu_iothread_add {
4390     my($vmid, $deviceid, $device) = @_;
4391
4392     if ($device->{iothread}) {
4393         my $iothreads = vm_iothreads_list($vmid);
4394         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4395     }
4396 }
4397
4398 sub qemu_iothread_del {
4399     my($conf, $vmid, $deviceid) = @_;
4400
4401     my $confid = $deviceid;
4402     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4403         $confid = 'scsi' . $1;
4404     }
4405     my $device = parse_drive($confid, $conf->{$confid});
4406     if ($device->{iothread}) {
4407         my $iothreads = vm_iothreads_list($vmid);
4408         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4409     }
4410 }
4411
4412 sub qemu_objectadd {
4413     my($vmid, $objectid, $qomtype) = @_;
4414
4415     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4416
4417     return 1;
4418 }
4419
4420 sub qemu_objectdel {
4421     my($vmid, $objectid) = @_;
4422
4423     vm_mon_cmd($vmid, "object-del", id => $objectid);
4424
4425     return 1;
4426 }
4427
4428 sub qemu_driveadd {
4429     my ($storecfg, $vmid, $device) = @_;
4430
4431     my $drive = print_drive_full($storecfg, $vmid, $device);
4432     $drive =~ s/\\/\\\\/g;
4433     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4434
4435     # If the command succeeds qemu prints: "OK"
4436     return 1 if $ret =~ m/OK/s;
4437
4438     die "adding drive failed: $ret\n";
4439 }
4440
4441 sub qemu_drivedel {
4442     my($vmid, $deviceid) = @_;
4443
4444     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4445     $ret =~ s/^\s+//;
4446
4447     return 1 if $ret eq "";
4448
4449     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4450     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4451
4452     die "deleting drive $deviceid failed : $ret\n";
4453 }
4454
4455 sub qemu_deviceaddverify {
4456     my ($vmid, $deviceid) = @_;
4457
4458     for (my $i = 0; $i <= 5; $i++) {
4459          my $devices_list = vm_devices_list($vmid);
4460          return 1 if defined($devices_list->{$deviceid});
4461          sleep 1;
4462     }
4463
4464     die "error on hotplug device '$deviceid'\n";
4465 }
4466
4467
4468 sub qemu_devicedelverify {
4469     my ($vmid, $deviceid) = @_;
4470
4471     # need to verify that the device is correctly removed as device_del
4472     # is async and empty return is not reliable
4473
4474     for (my $i = 0; $i <= 5; $i++) {
4475          my $devices_list = vm_devices_list($vmid);
4476          return 1 if !defined($devices_list->{$deviceid});
4477          sleep 1;
4478     }
4479
4480     die "error on hot-unplugging device '$deviceid'\n";
4481 }
4482
4483 sub qemu_findorcreatescsihw {
4484     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4485
4486     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4487
4488     my $scsihwid="$controller_prefix$controller";
4489     my $devices_list = vm_devices_list($vmid);
4490
4491     if(!defined($devices_list->{$scsihwid})) {
4492         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4493     }
4494
4495     return 1;
4496 }
4497
4498 sub qemu_deletescsihw {
4499     my ($conf, $vmid, $opt) = @_;
4500
4501     my $device = parse_drive($opt, $conf->{$opt});
4502
4503     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4504         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4505         return 1;
4506     }
4507
4508     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4509
4510     my $devices_list = vm_devices_list($vmid);
4511     foreach my $opt (keys %{$devices_list}) {
4512         if (PVE::QemuServer::is_valid_drivename($opt)) {
4513             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4514             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4515                 return 1;
4516             }
4517         }
4518     }
4519
4520     my $scsihwid="scsihw$controller";
4521
4522     vm_deviceunplug($vmid, $conf, $scsihwid);
4523
4524     return 1;
4525 }
4526
4527 sub qemu_add_pci_bridge {
4528     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4529
4530     my $bridges = {};
4531
4532     my $bridgeid;
4533
4534     print_pci_addr($device, $bridges, $arch, $machine_type);
4535
4536     while (my ($k, $v) = each %$bridges) {
4537         $bridgeid = $k;
4538     }
4539     return 1 if !defined($bridgeid) || $bridgeid < 1;
4540
4541     my $bridge = "pci.$bridgeid";
4542     my $devices_list = vm_devices_list($vmid);
4543
4544     if (!defined($devices_list->{$bridge})) {
4545         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4546     }
4547
4548     return 1;
4549 }
4550
4551 sub qemu_set_link_status {
4552     my ($vmid, $device, $up) = @_;
4553
4554     vm_mon_cmd($vmid, "set_link", name => $device,
4555                up => $up ? JSON::true : JSON::false);
4556 }
4557
4558 sub qemu_netdevadd {
4559     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4560
4561     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4562     my %options =  split(/[=,]/, $netdev);
4563
4564     vm_mon_cmd($vmid, "netdev_add",  %options);
4565     return 1;
4566 }
4567
4568 sub qemu_netdevdel {
4569     my ($vmid, $deviceid) = @_;
4570
4571     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4572 }
4573
4574 sub qemu_usb_hotplug {
4575     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4576
4577     return if !$device;
4578
4579     # remove the old one first
4580     vm_deviceunplug($vmid, $conf, $deviceid);
4581
4582     # check if xhci controller is necessary and available
4583     if ($device->{usb3}) {
4584
4585         my $devicelist = vm_devices_list($vmid);
4586
4587         if (!$devicelist->{xhci}) {
4588             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4589             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4590         }
4591     }
4592     my $d = parse_usb_device($device->{host});
4593     $d->{usb3} = $device->{usb3};
4594
4595     # add the new one
4596     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4597 }
4598
4599 sub qemu_cpu_hotplug {
4600     my ($vmid, $conf, $vcpus) = @_;
4601
4602     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4603
4604     my $sockets = 1;
4605     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4606     $sockets = $conf->{sockets} if  $conf->{sockets};
4607     my $cores = $conf->{cores} || 1;
4608     my $maxcpus = $sockets * $cores;
4609
4610     $vcpus = $maxcpus if !$vcpus;
4611
4612     die "you can't add more vcpus than maxcpus\n"
4613         if $vcpus > $maxcpus;
4614
4615     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4616
4617     if ($vcpus < $currentvcpus) {
4618
4619         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4620
4621             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4622                 qemu_devicedel($vmid, "cpu$i");
4623                 my $retry = 0;
4624                 my $currentrunningvcpus = undef;
4625                 while (1) {
4626                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4627                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4628                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4629                     $retry++;
4630                     sleep 1;
4631                 }
4632                 #update conf after each succesfull cpu unplug
4633                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4634                 PVE::QemuConfig->write_config($vmid, $conf);
4635             }
4636         } else {
4637             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4638         }
4639
4640         return;
4641     }
4642
4643     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4644     die "vcpus in running vm does not match its configuration\n"
4645         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4646
4647     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4648
4649         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4650             my $cpustr = print_cpu_device($conf, $i);
4651             qemu_deviceadd($vmid, $cpustr);
4652
4653             my $retry = 0;
4654             my $currentrunningvcpus = undef;
4655             while (1) {
4656                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4657                 last if scalar(@{$currentrunningvcpus}) == $i;
4658                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4659                 sleep 1;
4660                 $retry++;
4661             }
4662             #update conf after each succesfull cpu hotplug
4663             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4664             PVE::QemuConfig->write_config($vmid, $conf);
4665         }
4666     } else {
4667
4668         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4669             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4670         }
4671     }
4672 }
4673
4674 sub qemu_block_set_io_throttle {
4675     my ($vmid, $deviceid,
4676         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4677         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4678         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4679         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4680
4681     return if !check_running($vmid) ;
4682
4683     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4684         bps => int($bps),
4685         bps_rd => int($bps_rd),
4686         bps_wr => int($bps_wr),
4687         iops => int($iops),
4688         iops_rd => int($iops_rd),
4689         iops_wr => int($iops_wr),
4690         bps_max => int($bps_max),
4691         bps_rd_max => int($bps_rd_max),
4692         bps_wr_max => int($bps_wr_max),
4693         iops_max => int($iops_max),
4694         iops_rd_max => int($iops_rd_max),
4695         iops_wr_max => int($iops_wr_max),
4696         bps_max_length => int($bps_max_length),
4697         bps_rd_max_length => int($bps_rd_max_length),
4698         bps_wr_max_length => int($bps_wr_max_length),
4699         iops_max_length => int($iops_max_length),
4700         iops_rd_max_length => int($iops_rd_max_length),
4701         iops_wr_max_length => int($iops_wr_max_length),
4702     );
4703
4704 }
4705
4706 # old code, only used to shutdown old VM after update
4707 sub __read_avail {
4708     my ($fh, $timeout) = @_;
4709
4710     my $sel = new IO::Select;
4711     $sel->add($fh);
4712
4713     my $res = '';
4714     my $buf;
4715
4716     my @ready;
4717     while (scalar (@ready = $sel->can_read($timeout))) {
4718         my $count;
4719         if ($count = $fh->sysread($buf, 8192)) {
4720             if ($buf =~ /^(.*)\(qemu\) $/s) {
4721                 $res .= $1;
4722                 last;
4723             } else {
4724                 $res .= $buf;
4725             }
4726         } else {
4727             if (!defined($count)) {
4728                 die "$!\n";
4729             }
4730             last;
4731         }
4732     }
4733
4734     die "monitor read timeout\n" if !scalar(@ready);
4735
4736     return $res;
4737 }
4738
4739 sub qemu_block_resize {
4740     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4741
4742     my $running = check_running($vmid);
4743
4744     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4745
4746     return if !$running;
4747
4748     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4749
4750 }
4751
4752 sub qemu_volume_snapshot {
4753     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4754
4755     my $running = check_running($vmid);
4756
4757     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4758         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4759     } else {
4760         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4761     }
4762 }
4763
4764 sub qemu_volume_snapshot_delete {
4765     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4766
4767     my $running = check_running($vmid);
4768
4769     if($running) {
4770
4771         $running = undef;
4772         my $conf = PVE::QemuConfig->load_config($vmid);
4773         foreach_drive($conf, sub {
4774             my ($ds, $drive) = @_;
4775             $running = 1 if $drive->{file} eq $volid;
4776         });
4777     }
4778
4779     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4780         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4781     } else {
4782         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4783     }
4784 }
4785
4786 sub set_migration_caps {
4787     my ($vmid) = @_;
4788
4789     my $cap_ref = [];
4790
4791     my $enabled_cap = {
4792         "auto-converge" => 1,
4793         "xbzrle" => 1,
4794         "x-rdma-pin-all" => 0,
4795         "zero-blocks" => 0,
4796         "compress" => 0
4797     };
4798
4799     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4800
4801     for my $supported_capability (@$supported_capabilities) {
4802         push @$cap_ref, {
4803             capability => $supported_capability->{capability},
4804             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4805         };
4806     }
4807
4808     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4809 }
4810
4811 my $fast_plug_option = {
4812     'lock' => 1,
4813     'name' => 1,
4814     'onboot' => 1,
4815     'shares' => 1,
4816     'startup' => 1,
4817     'description' => 1,
4818     'protection' => 1,
4819     'vmstatestorage' => 1,
4820     'hookscript' => 1,
4821 };
4822
4823 # hotplug changes in [PENDING]
4824 # $selection hash can be used to only apply specified options, for
4825 # example: { cores => 1 } (only apply changed 'cores')
4826 # $errors ref is used to return error messages
4827 sub vmconfig_hotplug_pending {
4828     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4829
4830     my $defaults = load_defaults();
4831     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4832
4833     # commit values which do not have any impact on running VM first
4834     # Note: those option cannot raise errors, we we do not care about
4835     # $selection and always apply them.
4836
4837     my $add_error = sub {
4838         my ($opt, $msg) = @_;
4839         $errors->{$opt} = "hotplug problem - $msg";
4840     };
4841
4842     my $changes = 0;
4843     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4844         if ($fast_plug_option->{$opt}) {
4845             $conf->{$opt} = $conf->{pending}->{$opt};
4846             delete $conf->{pending}->{$opt};
4847             $changes = 1;
4848         }
4849     }
4850
4851     if ($changes) {
4852         PVE::QemuConfig->write_config($vmid, $conf);
4853         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4854     }
4855
4856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4857
4858     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4859     foreach my $opt (sort keys %$pending_delete_hash) {
4860         next if $selection && !$selection->{$opt};
4861         my $force = $pending_delete_hash->{$opt}->{force};
4862         eval {
4863             if ($opt eq 'hotplug') {
4864                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4865             } elsif ($opt eq 'tablet') {
4866                 die "skip\n" if !$hotplug_features->{usb};
4867                 if ($defaults->{tablet}) {
4868                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4870                         if $arch eq 'aarch64';
4871                 } else {
4872                     vm_deviceunplug($vmid, $conf, 'tablet');
4873                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4874                 }
4875             } elsif ($opt =~ m/^usb\d+/) {
4876                 die "skip\n";
4877                 # since we cannot reliably hot unplug usb devices
4878                 # we are disabling it
4879                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4880                 vm_deviceunplug($vmid, $conf, $opt);
4881             } elsif ($opt eq 'vcpus') {
4882                 die "skip\n" if !$hotplug_features->{cpu};
4883                 qemu_cpu_hotplug($vmid, $conf, undef);
4884             } elsif ($opt eq 'balloon') {
4885                 # enable balloon device is not hotpluggable
4886                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4887                 # here we reset the ballooning value to memory
4888                 my $balloon = $conf->{memory} || $defaults->{memory};
4889                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4890             } elsif ($fast_plug_option->{$opt}) {
4891                 # do nothing
4892             } elsif ($opt =~ m/^net(\d+)$/) {
4893                 die "skip\n" if !$hotplug_features->{network};
4894                 vm_deviceunplug($vmid, $conf, $opt);
4895             } elsif (is_valid_drivename($opt)) {
4896                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4897                 vm_deviceunplug($vmid, $conf, $opt);
4898                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4899             } elsif ($opt =~ m/^memory$/) {
4900                 die "skip\n" if !$hotplug_features->{memory};
4901                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4902             } elsif ($opt eq 'cpuunits') {
4903                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4904             } elsif ($opt eq 'cpulimit') {
4905                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4906             } else {
4907                 die "skip\n";
4908             }
4909         };
4910         if (my $err = $@) {
4911             &$add_error($opt, $err) if $err ne "skip\n";
4912         } else {
4913             # save new config if hotplug was successful
4914             delete $conf->{$opt};
4915             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4916             PVE::QemuConfig->write_config($vmid, $conf);
4917             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4918         }
4919     }
4920
4921     my $apply_pending_cloudinit;
4922     $apply_pending_cloudinit = sub {
4923         my ($key, $value) = @_;
4924         $apply_pending_cloudinit = sub {}; # once is enough
4925
4926         my @cloudinit_opts = keys %$confdesc_cloudinit;
4927         foreach my $opt (keys %{$conf->{pending}}) {
4928             next if !grep { $_ eq $opt } @cloudinit_opts;
4929             $conf->{$opt} = delete $conf->{pending}->{$opt};
4930         }
4931
4932         my $new_conf = { %$conf };
4933         $new_conf->{$key} = $value;
4934         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4935     };
4936
4937     foreach my $opt (keys %{$conf->{pending}}) {
4938         next if $selection && !$selection->{$opt};
4939         my $value = $conf->{pending}->{$opt};
4940         eval {
4941             if ($opt eq 'hotplug') {
4942                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4943             } elsif ($opt eq 'tablet') {
4944                 die "skip\n" if !$hotplug_features->{usb};
4945                 if ($value == 1) {
4946                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4947                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4948                         if $arch eq 'aarch64';
4949                 } elsif ($value == 0) {
4950                     vm_deviceunplug($vmid, $conf, 'tablet');
4951                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4952                 }
4953             } elsif ($opt =~ m/^usb\d+$/) {
4954                 die "skip\n";
4955                 # since we cannot reliably hot unplug usb devices
4956                 # we are disabling it
4957                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4958                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4959                 die "skip\n" if !$d;
4960                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4961             } elsif ($opt eq 'vcpus') {
4962                 die "skip\n" if !$hotplug_features->{cpu};
4963                 qemu_cpu_hotplug($vmid, $conf, $value);
4964             } elsif ($opt eq 'balloon') {
4965                 # enable/disable balloning device is not hotpluggable
4966                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4967                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4968                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4969
4970                 # allow manual ballooning if shares is set to zero
4971                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4972                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4973                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4974                 }
4975             } elsif ($opt =~ m/^net(\d+)$/) {
4976                 # some changes can be done without hotplug
4977                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4978                                     $vmid, $opt, $value, $arch, $machine_type);
4979             } elsif (is_valid_drivename($opt)) {
4980                 # some changes can be done without hotplug
4981                 my $drive = parse_drive($opt, $value);
4982                 if (drive_is_cloudinit($drive)) {
4983                     &$apply_pending_cloudinit($opt, $value);
4984                 }
4985                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4986                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4987             } elsif ($opt =~ m/^memory$/) { #dimms
4988                 die "skip\n" if !$hotplug_features->{memory};
4989                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4990             } elsif ($opt eq 'cpuunits') {
4991                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4992             } elsif ($opt eq 'cpulimit') {
4993                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4994                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4995             } else {
4996                 die "skip\n";  # skip non-hot-pluggable options
4997             }
4998         };
4999         if (my $err = $@) {
5000             &$add_error($opt, $err) if $err ne "skip\n";
5001         } else {
5002             # save new config if hotplug was successful
5003             $conf->{$opt} = $value;
5004             delete $conf->{pending}->{$opt};
5005             PVE::QemuConfig->write_config($vmid, $conf);
5006             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5007         }
5008     }
5009 }
5010
5011 sub try_deallocate_drive {
5012     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5013
5014     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5015         my $volid = $drive->{file};
5016         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5017             my $sid = PVE::Storage::parse_volume_id($volid);
5018             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5019
5020             # check if the disk is really unused
5021             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5022                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5023             PVE::Storage::vdisk_free($storecfg, $volid);
5024             return 1;
5025         } else {
5026             # If vm is not owner of this disk remove from config
5027             return 1;
5028         }
5029     }
5030
5031     return undef;
5032 }
5033
5034 sub vmconfig_delete_or_detach_drive {
5035     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5036
5037     my $drive = parse_drive($opt, $conf->{$opt});
5038
5039     my $rpcenv = PVE::RPCEnvironment::get();
5040     my $authuser = $rpcenv->get_user();
5041
5042     if ($force) {
5043         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5044         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5045     } else {
5046         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5047     }
5048 }
5049
5050
5051
5052 sub vmconfig_apply_pending {
5053     my ($vmid, $conf, $storecfg) = @_;
5054
5055     # cold plug
5056
5057     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5058     foreach my $opt (sort keys %$pending_delete_hash) {
5059         die "internal error" if $opt =~ m/^unused/;
5060         my $force = $pending_delete_hash->{$opt}->{force};
5061         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5062         if (!defined($conf->{$opt})) {
5063             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5064             PVE::QemuConfig->write_config($vmid, $conf);
5065         } elsif (is_valid_drivename($opt)) {
5066             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5067             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5068             delete $conf->{$opt};
5069             PVE::QemuConfig->write_config($vmid, $conf);
5070         } else {
5071             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5072             delete $conf->{$opt};
5073             PVE::QemuConfig->write_config($vmid, $conf);
5074         }
5075     }
5076
5077     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5078
5079     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5080         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5081
5082         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5083             # skip if nothing changed
5084         } elsif (is_valid_drivename($opt)) {
5085             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5086                 if defined($conf->{$opt});
5087             $conf->{$opt} = $conf->{pending}->{$opt};
5088         } else {
5089             $conf->{$opt} = $conf->{pending}->{$opt};
5090         }
5091
5092         delete $conf->{pending}->{$opt};
5093         PVE::QemuConfig->write_config($vmid, $conf);
5094     }
5095 }
5096
5097 my $safe_num_ne = sub {
5098     my ($a, $b) = @_;
5099
5100     return 0 if !defined($a) && !defined($b);
5101     return 1 if !defined($a);
5102     return 1 if !defined($b);
5103
5104     return $a != $b;
5105 };
5106
5107 my $safe_string_ne = sub {
5108     my ($a, $b) = @_;
5109
5110     return 0 if !defined($a) && !defined($b);
5111     return 1 if !defined($a);
5112     return 1 if !defined($b);
5113
5114     return $a ne $b;
5115 };
5116
5117 sub vmconfig_update_net {
5118     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5119
5120     my $newnet = parse_net($value);
5121
5122     if ($conf->{$opt}) {
5123         my $oldnet = parse_net($conf->{$opt});
5124
5125         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5126             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5127             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5128             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5129
5130             # for non online change, we try to hot-unplug
5131             die "skip\n" if !$hotplug;
5132             vm_deviceunplug($vmid, $conf, $opt);
5133         } else {
5134
5135             die "internal error" if $opt !~ m/net(\d+)/;
5136             my $iface = "tap${vmid}i$1";
5137
5138             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5139                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5140                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5141                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5142                 PVE::Network::tap_unplug($iface);
5143                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5144             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5145                 # Rate can be applied on its own but any change above needs to
5146                 # include the rate in tap_plug since OVS resets everything.
5147                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5148             }
5149
5150             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5151                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5152             }
5153
5154             return 1;
5155         }
5156     }
5157
5158     if ($hotplug) {
5159         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5160     } else {
5161         die "skip\n";
5162     }
5163 }
5164
5165 sub vmconfig_update_disk {
5166     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5167
5168     # fixme: do we need force?
5169
5170     my $drive = parse_drive($opt, $value);
5171
5172     if ($conf->{$opt}) {
5173
5174         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5175
5176             my $media = $drive->{media} || 'disk';
5177             my $oldmedia = $old_drive->{media} || 'disk';
5178             die "unable to change media type\n" if $media ne $oldmedia;
5179
5180             if (!drive_is_cdrom($old_drive)) {
5181
5182                 if ($drive->{file} ne $old_drive->{file}) {
5183
5184                     die "skip\n" if !$hotplug;
5185
5186                     # unplug and register as unused
5187                     vm_deviceunplug($vmid, $conf, $opt);
5188                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5189
5190                 } else {
5191                     # update existing disk
5192
5193                     # skip non hotpluggable value
5194                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5195                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5196                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5197                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5198                         die "skip\n";
5199                     }
5200
5201                     # apply throttle
5202                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5203                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5204                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5205                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5206                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5207                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5208                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5209                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5210                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5211                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5212                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5213                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5214                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5215                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5216                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5217                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5218                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5219                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5220
5221                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5222                                                    ($drive->{mbps} || 0)*1024*1024,
5223                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5224                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5225                                                    $drive->{iops} || 0,
5226                                                    $drive->{iops_rd} || 0,
5227                                                    $drive->{iops_wr} || 0,
5228                                                    ($drive->{mbps_max} || 0)*1024*1024,
5229                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5230                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5231                                                    $drive->{iops_max} || 0,
5232                                                    $drive->{iops_rd_max} || 0,
5233                                                    $drive->{iops_wr_max} || 0,
5234                                                    $drive->{bps_max_length} || 1,
5235                                                    $drive->{bps_rd_max_length} || 1,
5236                                                    $drive->{bps_wr_max_length} || 1,
5237                                                    $drive->{iops_max_length} || 1,
5238                                                    $drive->{iops_rd_max_length} || 1,
5239                                                    $drive->{iops_wr_max_length} || 1);
5240
5241                     }
5242
5243                     return 1;
5244                 }
5245
5246             } else { # cdrom
5247
5248                 if ($drive->{file} eq 'none') {
5249                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5250                     if (drive_is_cloudinit($old_drive)) {
5251                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5252                     }
5253                 } else {
5254                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5255                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5256                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5257                 }
5258
5259                 return 1;
5260             }
5261         }
5262     }
5263
5264     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5265     # hotplug new disks
5266     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5267     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5268 }
5269
5270 sub vm_start {
5271     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5272         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5273
5274     PVE::QemuConfig->lock_config($vmid, sub {
5275         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5276
5277         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5278
5279         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5280
5281         PVE::QemuConfig->check_lock($conf)
5282             if !($skiplock || $is_suspended);
5283
5284         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5285
5286         # clean up leftover reboot request files
5287         eval { clear_reboot_request($vmid); };
5288         warn $@ if $@;
5289
5290         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5291             vmconfig_apply_pending($vmid, $conf, $storecfg);
5292             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5293         }
5294
5295         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5296
5297         my $defaults = load_defaults();
5298
5299         # set environment variable useful inside network script
5300         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5301
5302         my $local_volumes = {};
5303
5304         if ($targetstorage) {
5305             foreach_drive($conf, sub {
5306                 my ($ds, $drive) = @_;
5307
5308                 return if drive_is_cdrom($drive);
5309
5310                 my $volid = $drive->{file};
5311
5312                 return if !$volid;
5313
5314                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5315
5316                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5317                 return if $scfg->{shared};
5318                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5319             });
5320
5321             my $format = undef;
5322
5323             foreach my $opt (sort keys %$local_volumes) {
5324
5325                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5326                 my $drive = parse_drive($opt, $conf->{$opt});
5327
5328                 #if remote storage is specified, use default format
5329                 if ($targetstorage && $targetstorage ne "1") {
5330                     $storeid = $targetstorage;
5331                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5332                     $format = $defFormat;
5333                 } else {
5334                     #else we use same format than original
5335                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5336                     $format = qemu_img_format($scfg, $volid);
5337                 }
5338
5339                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5340                 my $newdrive = $drive;
5341                 $newdrive->{format} = $format;
5342                 $newdrive->{file} = $newvolid;
5343                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5344                 $local_volumes->{$opt} = $drivestr;
5345                 #pass drive to conf for command line
5346                 $conf->{$opt} = $drivestr;
5347             }
5348         }
5349
5350         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5351
5352         if ($is_suspended) {
5353             # enforce machine type on suspended vm to ensure HW compatibility
5354             $forcemachine = $conf->{runningmachine};
5355             print "Resuming suspended VM\n";
5356         }
5357
5358         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5359
5360         my $migrate_uri;
5361         if ($statefile) {
5362             if ($statefile eq 'tcp') {
5363                 my $localip = "localhost";
5364                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5365                 my $nodename = PVE::INotify::nodename();
5366
5367                 if (!defined($migration_type)) {
5368                     if (defined($datacenterconf->{migration}->{type})) {
5369                         $migration_type = $datacenterconf->{migration}->{type};
5370                     } else {
5371                         $migration_type = 'secure';
5372                     }
5373                 }
5374
5375                 if ($migration_type eq 'insecure') {
5376                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5377                     if ($migrate_network_addr) {
5378                         $localip = $migrate_network_addr;
5379                     } else {
5380                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5381                     }
5382
5383                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5384                 }
5385
5386                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5387                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5388                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5389                 push @$cmd, '-incoming', $migrate_uri;
5390                 push @$cmd, '-S';
5391
5392             } elsif ($statefile eq 'unix') {
5393                 # should be default for secure migrations as a ssh TCP forward
5394                 # tunnel is not deterministic reliable ready and fails regurarly
5395                 # to set up in time, so use UNIX socket forwards
5396                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5397                 unlink $socket_addr;
5398
5399                 $migrate_uri = "unix:$socket_addr";
5400
5401                 push @$cmd, '-incoming', $migrate_uri;
5402                 push @$cmd, '-S';
5403
5404             } elsif (-e $statefile) {
5405                 push @$cmd, '-loadstate', $statefile;
5406             } else {
5407                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5408                 push @$vollist, $statefile;
5409                 push @$cmd, '-loadstate', $statepath;
5410             }
5411         } elsif ($paused) {
5412             push @$cmd, '-S';
5413         }
5414
5415         # host pci devices
5416         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5417           my $d = parse_hostpci($conf->{"hostpci$i"});
5418           next if !$d;
5419           my $pcidevices = $d->{pciid};
5420           foreach my $pcidevice (@$pcidevices) {
5421                 my $pciid = $pcidevice->{id};
5422
5423                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5424                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5425                 die "no pci device info for device '$pciid'\n" if !$info;
5426
5427                 if ($d->{mdev}) {
5428                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5429                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5430                 } else {
5431                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5432                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5433                     die "can't reset pci device '$pciid'\n"
5434                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5435                 }
5436           }
5437         }
5438
5439         PVE::Storage::activate_volumes($storecfg, $vollist);
5440
5441         eval {
5442             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5443                 outfunc => sub {}, errfunc => sub {});
5444         };
5445         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5446         # timeout should be more than enough here...
5447         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5448
5449         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5450                                                   : $defaults->{cpuunits};
5451
5452         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5453         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5454
5455         my %properties = (
5456             Slice => 'qemu.slice',
5457             KillMode => 'none',
5458             CPUShares => $cpuunits
5459         );
5460
5461         if (my $cpulimit = $conf->{cpulimit}) {
5462             $properties{CPUQuota} = int($cpulimit * 100);
5463         }
5464         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5465
5466         my $run_qemu = sub {
5467             PVE::Tools::run_fork sub {
5468                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5469                 run_command($cmd, %run_params);
5470             };
5471         };
5472
5473         if ($conf->{hugepages}) {
5474
5475             my $code = sub {
5476                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5477                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5478
5479                 PVE::QemuServer::Memory::hugepages_mount();
5480                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5481
5482                 eval { $run_qemu->() };
5483                 if (my $err = $@) {
5484                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5485                     die $err;
5486                 }
5487
5488                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5489             };
5490             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5491
5492         } else {
5493             eval { $run_qemu->() };
5494         }
5495
5496         if (my $err = $@) {
5497             # deactivate volumes if start fails
5498             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5499             die "start failed: $err";
5500         }
5501
5502         print "migration listens on $migrate_uri\n" if $migrate_uri;
5503
5504         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5505             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5506             warn $@ if $@;
5507         }
5508
5509         #start nbd server for storage migration
5510         if ($targetstorage) {
5511             my $nodename = PVE::INotify::nodename();
5512             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5513             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5514             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5515             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5516
5517             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5518
5519             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5520
5521             foreach my $opt (sort keys %$local_volumes) {
5522                 my $volid = $local_volumes->{$opt};
5523                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5524                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5525                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5526             }
5527         }
5528
5529         if ($migratedfrom) {
5530             eval {
5531                 set_migration_caps($vmid);
5532             };
5533             warn $@ if $@;
5534
5535             if ($spice_port) {
5536                 print "spice listens on port $spice_port\n";
5537                 if ($spice_ticket) {
5538                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5539                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5540                 }
5541             }
5542
5543         } else {
5544             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5545                 if !$statefile && $conf->{balloon};
5546
5547             foreach my $opt (keys %$conf) {
5548                 next if $opt !~  m/^net\d+$/;
5549                 my $nicconf = parse_net($conf->{$opt});
5550                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5551             }
5552         }
5553
5554         vm_mon_cmd_nocheck($vmid, 'qom-set',
5555                     path => "machine/peripheral/balloon0",
5556                     property => "guest-stats-polling-interval",
5557                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5558
5559         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5560             print "Resumed VM, removing state\n";
5561             delete $conf->@{qw(lock vmstate runningmachine)};
5562             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5563             PVE::Storage::vdisk_free($storecfg, $vmstate);
5564             PVE::QemuConfig->write_config($vmid, $conf);
5565         }
5566
5567         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5568     });
5569 }
5570
5571 sub vm_mon_cmd {
5572     my ($vmid, $execute, %params) = @_;
5573
5574     my $cmd = { execute => $execute, arguments => \%params };
5575     vm_qmp_command($vmid, $cmd);
5576 }
5577
5578 sub vm_mon_cmd_nocheck {
5579     my ($vmid, $execute, %params) = @_;
5580
5581     my $cmd = { execute => $execute, arguments => \%params };
5582     vm_qmp_command($vmid, $cmd, 1);
5583 }
5584
5585 sub vm_qmp_command {
5586     my ($vmid, $cmd, $nocheck) = @_;
5587
5588     my $res;
5589
5590     my $timeout;
5591     if ($cmd->{arguments}) {
5592         $timeout = delete $cmd->{arguments}->{timeout};
5593     }
5594
5595     eval {
5596         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5597         my $sname = qmp_socket($vmid);
5598         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5599             my $qmpclient = PVE::QMPClient->new();
5600
5601             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5602         } else {
5603             die "unable to open monitor socket\n";
5604         }
5605     };
5606     if (my $err = $@) {
5607         syslog("err", "VM $vmid qmp command failed - $err");
5608         die $err;
5609     }
5610
5611     return $res;
5612 }
5613
5614 sub vm_human_monitor_command {
5615     my ($vmid, $cmdline) = @_;
5616
5617     my $cmd = {
5618         execute => 'human-monitor-command',
5619         arguments => { 'command-line' => $cmdline},
5620     };
5621
5622     return vm_qmp_command($vmid, $cmd);
5623 }
5624
5625 sub vm_commandline {
5626     my ($storecfg, $vmid, $snapname) = @_;
5627
5628     my $conf = PVE::QemuConfig->load_config($vmid);
5629
5630     if ($snapname) {
5631         my $snapshot = $conf->{snapshots}->{$snapname};
5632         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5633
5634         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5635
5636         $conf = $snapshot;
5637     }
5638
5639     my $defaults = load_defaults();
5640
5641     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5642
5643     return PVE::Tools::cmd2string($cmd);
5644 }
5645
5646 sub vm_reset {
5647     my ($vmid, $skiplock) = @_;
5648
5649     PVE::QemuConfig->lock_config($vmid, sub {
5650
5651         my $conf = PVE::QemuConfig->load_config($vmid);
5652
5653         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5654
5655         vm_mon_cmd($vmid, "system_reset");
5656     });
5657 }
5658
5659 sub get_vm_volumes {
5660     my ($conf) = @_;
5661
5662     my $vollist = [];
5663     foreach_volid($conf, sub {
5664         my ($volid, $attr) = @_;
5665
5666         return if $volid =~ m|^/|;
5667
5668         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5669         return if !$sid;
5670
5671         push @$vollist, $volid;
5672     });
5673
5674     return $vollist;
5675 }
5676
5677 sub vm_stop_cleanup {
5678     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5679
5680     eval {
5681
5682         if (!$keepActive) {
5683             my $vollist = get_vm_volumes($conf);
5684             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5685         }
5686
5687         foreach my $ext (qw(mon qmp pid vnc qga)) {
5688             unlink "/var/run/qemu-server/${vmid}.$ext";
5689         }
5690
5691         if ($conf->{ivshmem}) {
5692             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5693             # just delete it for now, VMs which have this already open do not
5694             # are affected, but new VMs will get a separated one. If this
5695             # becomes an issue we either add some sort of ref-counting or just
5696             # add a "don't delete on stop" flag to the ivshmem format.
5697             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5698         }
5699
5700         foreach my $key (keys %$conf) {
5701             next if $key !~ m/^hostpci(\d+)$/;
5702             my $hostpciindex = $1;
5703             my $d = parse_hostpci($conf->{$key});
5704             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5705
5706             foreach my $pci (@{$d->{pciid}}) {
5707                 my $pciid = $pci->{id};
5708                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5709             }
5710         }
5711
5712         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5713     };
5714     warn $@ if $@; # avoid errors - just warn
5715 }
5716
5717 # call only in locked context
5718 sub _do_vm_stop {
5719     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5720
5721     my $pid = check_running($vmid, $nocheck);
5722     return if !$pid;
5723
5724     my $conf;
5725     if (!$nocheck) {
5726         $conf = PVE::QemuConfig->load_config($vmid);
5727         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5728         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5729             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5730             $timeout = $opts->{down} if $opts->{down};
5731         }
5732         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5733     }
5734
5735     eval {
5736         if ($shutdown) {
5737             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5738                 vm_qmp_command($vmid, {
5739                         execute => "guest-shutdown",
5740                         arguments => { timeout => $timeout }
5741                     }, $nocheck);
5742             } else {
5743                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5744             }
5745         } else {
5746             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5747         }
5748     };
5749     my $err = $@;
5750
5751     if (!$err) {
5752         $timeout = 60 if !defined($timeout);
5753
5754         my $count = 0;
5755         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5756             $count++;
5757             sleep 1;
5758         }
5759
5760         if ($count >= $timeout) {
5761             if ($force) {
5762                 warn "VM still running - terminating now with SIGTERM\n";
5763                 kill 15, $pid;
5764             } else {
5765                 die "VM quit/powerdown failed - got timeout\n";
5766             }
5767         } else {
5768             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5769             return;
5770         }
5771     } else {
5772         if ($force) {
5773             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5774             kill 15, $pid;
5775         } else {
5776             die "VM quit/powerdown failed\n";
5777         }
5778     }
5779
5780     # wait again
5781     $timeout = 10;
5782
5783     my $count = 0;
5784     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5785         $count++;
5786         sleep 1;
5787     }
5788
5789     if ($count >= $timeout) {
5790         warn "VM still running - terminating now with SIGKILL\n";
5791         kill 9, $pid;
5792         sleep 1;
5793     }
5794
5795     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5796 }
5797
5798 # Note: use $nocheck to skip tests if VM configuration file exists.
5799 # We need that when migration VMs to other nodes (files already moved)
5800 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5801 sub vm_stop {
5802     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5803
5804     $force = 1 if !defined($force) && !$shutdown;
5805
5806     if ($migratedfrom){
5807         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5808         kill 15, $pid if $pid;
5809         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5810         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5811         return;
5812     }
5813
5814     PVE::QemuConfig->lock_config($vmid, sub {
5815         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5816    });
5817 }
5818
5819 sub vm_reboot {
5820     my ($vmid, $timeout) = @_;
5821
5822     PVE::QemuConfig->lock_config($vmid, sub {
5823
5824         # only reboot if running, as qmeventd starts it again on a stop event
5825         return if !check_running($vmid);
5826
5827         create_reboot_request($vmid);
5828
5829         my $storecfg = PVE::Storage::config();
5830         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5831
5832    });
5833 }
5834
5835 sub vm_suspend {
5836     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5837
5838     my $conf;
5839     my $path;
5840     my $storecfg;
5841     my $vmstate;
5842
5843     PVE::QemuConfig->lock_config($vmid, sub {
5844
5845         $conf = PVE::QemuConfig->load_config($vmid);
5846
5847         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5848         PVE::QemuConfig->check_lock($conf)
5849             if !($skiplock || $is_backing_up);
5850
5851         die "cannot suspend to disk during backup\n"
5852             if $is_backing_up && $includestate;
5853
5854         if ($includestate) {
5855             $conf->{lock} = 'suspending';
5856             my $date = strftime("%Y-%m-%d", localtime(time()));
5857             $storecfg = PVE::Storage::config();
5858             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5859             $path = PVE::Storage::path($storecfg, $vmstate);
5860             PVE::QemuConfig->write_config($vmid, $conf);
5861         } else {
5862             vm_mon_cmd($vmid, "stop");
5863         }
5864     });
5865
5866     if ($includestate) {
5867         # save vm state
5868         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5869
5870         eval {
5871             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5872             for(;;) {
5873                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5874                 if (!$state->{status}) {
5875                     die "savevm not active\n";
5876                 } elsif ($state->{status} eq 'active') {
5877                     sleep(1);
5878                     next;
5879                 } elsif ($state->{status} eq 'completed') {
5880                     print "State saved, quitting\n";
5881                     last;
5882                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5883                     die "query-savevm failed with error '$state->{error}'\n"
5884                 } else {
5885                     die "query-savevm returned status '$state->{status}'\n";
5886                 }
5887             }
5888         };
5889         my $err = $@;
5890
5891         PVE::QemuConfig->lock_config($vmid, sub {
5892             $conf = PVE::QemuConfig->load_config($vmid);
5893             if ($err) {
5894                 # cleanup, but leave suspending lock, to indicate something went wrong
5895                 eval {
5896                     vm_mon_cmd($vmid, "savevm-end");
5897                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5898                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5899                     delete $conf->@{qw(vmstate runningmachine)};
5900                     PVE::QemuConfig->write_config($vmid, $conf);
5901                 };
5902                 warn $@ if $@;
5903                 die $err;
5904             }
5905
5906             die "lock changed unexpectedly\n"
5907                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5908
5909             vm_qmp_command($vmid, { execute => "quit" });
5910             $conf->{lock} = 'suspended';
5911             PVE::QemuConfig->write_config($vmid, $conf);
5912         });
5913     }
5914 }
5915
5916 sub vm_resume {
5917     my ($vmid, $skiplock, $nocheck) = @_;
5918
5919     PVE::QemuConfig->lock_config($vmid, sub {
5920         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5921         my $res = $vm_mon_cmd->($vmid, 'query-status');
5922         my $resume_cmd = 'cont';
5923
5924         if ($res->{status} && $res->{status} eq 'suspended') {
5925             $resume_cmd = 'system_wakeup';
5926         }
5927
5928         if (!$nocheck) {
5929
5930             my $conf = PVE::QemuConfig->load_config($vmid);
5931
5932             PVE::QemuConfig->check_lock($conf)
5933                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5934         }
5935
5936         $vm_mon_cmd->($vmid, $resume_cmd);
5937     });
5938 }
5939
5940 sub vm_sendkey {
5941     my ($vmid, $skiplock, $key) = @_;
5942
5943     PVE::QemuConfig->lock_config($vmid, sub {
5944
5945         my $conf = PVE::QemuConfig->load_config($vmid);
5946
5947         # there is no qmp command, so we use the human monitor command
5948         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5949         die $res if $res ne '';
5950     });
5951 }
5952
5953 sub vm_destroy {
5954     my ($storecfg, $vmid, $skiplock) = @_;
5955
5956     PVE::QemuConfig->lock_config($vmid, sub {
5957
5958         my $conf = PVE::QemuConfig->load_config($vmid);
5959
5960         if (!check_running($vmid)) {
5961             destroy_vm($storecfg, $vmid, undef, $skiplock);
5962         } else {
5963             die "VM $vmid is running - destroy failed\n";
5964         }
5965     });
5966 }
5967
5968 # vzdump restore implementaion
5969
5970 sub tar_archive_read_firstfile {
5971     my $archive = shift;
5972
5973     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5974
5975     # try to detect archive type first
5976     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5977         die "unable to open file '$archive'\n";
5978     my $firstfile = <$fh>;
5979     kill 15, $pid;
5980     close $fh;
5981
5982     die "ERROR: archive contaions no data\n" if !$firstfile;
5983     chomp $firstfile;
5984
5985     return $firstfile;
5986 }
5987
5988 sub tar_restore_cleanup {
5989     my ($storecfg, $statfile) = @_;
5990
5991     print STDERR "starting cleanup\n";
5992
5993     if (my $fd = IO::File->new($statfile, "r")) {
5994         while (defined(my $line = <$fd>)) {
5995             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5996                 my $volid = $2;
5997                 eval {
5998                     if ($volid =~ m|^/|) {
5999                         unlink $volid || die 'unlink failed\n';
6000                     } else {
6001                         PVE::Storage::vdisk_free($storecfg, $volid);
6002                     }
6003                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6004                 };
6005                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6006             } else {
6007                 print STDERR "unable to parse line in statfile - $line";
6008             }
6009         }
6010         $fd->close();
6011     }
6012 }
6013
6014 sub restore_archive {
6015     my ($archive, $vmid, $user, $opts) = @_;
6016
6017     my $format = $opts->{format};
6018     my $comp;
6019
6020     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6021         $format = 'tar' if !$format;
6022         $comp = 'gzip';
6023     } elsif ($archive =~ m/\.tar$/) {
6024         $format = 'tar' if !$format;
6025     } elsif ($archive =~ m/.tar.lzo$/) {
6026         $format = 'tar' if !$format;
6027         $comp = 'lzop';
6028     } elsif ($archive =~ m/\.vma$/) {
6029         $format = 'vma' if !$format;
6030     } elsif ($archive =~ m/\.vma\.gz$/) {
6031         $format = 'vma' if !$format;
6032         $comp = 'gzip';
6033     } elsif ($archive =~ m/\.vma\.lzo$/) {
6034         $format = 'vma' if !$format;
6035         $comp = 'lzop';
6036     } else {
6037         $format = 'vma' if !$format; # default
6038     }
6039
6040     # try to detect archive format
6041     if ($format eq 'tar') {
6042         return restore_tar_archive($archive, $vmid, $user, $opts);
6043     } else {
6044         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6045     }
6046 }
6047
6048 sub restore_update_config_line {
6049     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6050
6051     return if $line =~ m/^\#qmdump\#/;
6052     return if $line =~ m/^\#vzdump\#/;
6053     return if $line =~ m/^lock:/;
6054     return if $line =~ m/^unused\d+:/;
6055     return if $line =~ m/^parent:/;
6056
6057     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6058     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6059         # try to convert old 1.X settings
6060         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6061         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6062             my ($model, $macaddr) = split(/\=/, $devconfig);
6063             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6064             my $net = {
6065                 model => $model,
6066                 bridge => "vmbr$ind",
6067                 macaddr => $macaddr,
6068             };
6069             my $netstr = print_net($net);
6070
6071             print $outfd "net$cookie->{netcount}: $netstr\n";
6072             $cookie->{netcount}++;
6073         }
6074     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6075         my ($id, $netstr) = ($1, $2);
6076         my $net = parse_net($netstr);
6077         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6078         $netstr = print_net($net);
6079         print $outfd "$id: $netstr\n";
6080     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6081         my $virtdev = $1;
6082         my $value = $3;
6083         my $di = parse_drive($virtdev, $value);
6084         if (defined($di->{backup}) && !$di->{backup}) {
6085             print $outfd "#$line";
6086         } elsif ($map->{$virtdev}) {
6087             delete $di->{format}; # format can change on restore
6088             $di->{file} = $map->{$virtdev};
6089             $value = print_drive($vmid, $di);
6090             print $outfd "$virtdev: $value\n";
6091         } else {
6092             print $outfd $line;
6093         }
6094     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6095         my $vmgenid = $1;
6096         if ($vmgenid ne '0') {
6097             # always generate a new vmgenid if there was a valid one setup
6098             $vmgenid = generate_uuid();
6099         }
6100         print $outfd "vmgenid: $vmgenid\n";
6101     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6102         my ($uuid, $uuid_str);
6103         UUID::generate($uuid);
6104         UUID::unparse($uuid, $uuid_str);
6105         my $smbios1 = parse_smbios1($2);
6106         $smbios1->{uuid} = $uuid_str;
6107         print $outfd $1.print_smbios1($smbios1)."\n";
6108     } else {
6109         print $outfd $line;
6110     }
6111 }
6112
6113 sub scan_volids {
6114     my ($cfg, $vmid) = @_;
6115
6116     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6117
6118     my $volid_hash = {};
6119     foreach my $storeid (keys %$info) {
6120         foreach my $item (@{$info->{$storeid}}) {
6121             next if !($item->{volid} && $item->{size});
6122             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6123             $volid_hash->{$item->{volid}} = $item;
6124         }
6125     }
6126
6127     return $volid_hash;
6128 }
6129
6130 sub is_volume_in_use {
6131     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6132
6133     my $path = PVE::Storage::path($storecfg, $volid);
6134
6135     my $scan_config = sub {
6136         my ($cref, $snapname) = @_;
6137
6138         foreach my $key (keys %$cref) {
6139             my $value = $cref->{$key};
6140             if (is_valid_drivename($key)) {
6141                 next if $skip_drive && $key eq $skip_drive;
6142                 my $drive = parse_drive($key, $value);
6143                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6144                 return 1 if $volid eq $drive->{file};
6145                 if ($drive->{file} =~ m!^/!) {
6146                     return 1 if $drive->{file} eq $path;
6147                 } else {
6148                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6149                     next if !$storeid;
6150                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6151                     next if !$scfg;
6152                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6153                 }
6154             }
6155         }
6156
6157         return 0;
6158     };
6159
6160     return 1 if &$scan_config($conf);
6161
6162     undef $skip_drive;
6163
6164     foreach my $snapname (keys %{$conf->{snapshots}}) {
6165         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6166     }
6167
6168     return 0;
6169 }
6170
6171 sub update_disksize {
6172     my ($vmid, $conf, $volid_hash) = @_;
6173
6174     my $changes;
6175     my $prefix = "VM $vmid:";
6176
6177     # used and unused disks
6178     my $referenced = {};
6179
6180     # Note: it is allowed to define multiple storages with same path (alias), so
6181     # we need to check both 'volid' and real 'path' (two different volid can point
6182     # to the same path).
6183
6184     my $referencedpath = {};
6185
6186     # update size info
6187     foreach my $opt (keys %$conf) {
6188         if (is_valid_drivename($opt)) {
6189             my $drive = parse_drive($opt, $conf->{$opt});
6190             my $volid = $drive->{file};
6191             next if !$volid;
6192
6193             $referenced->{$volid} = 1;
6194             if ($volid_hash->{$volid} &&
6195                 (my $path = $volid_hash->{$volid}->{path})) {
6196                 $referencedpath->{$path} = 1;
6197             }
6198
6199             next if drive_is_cdrom($drive);
6200             next if !$volid_hash->{$volid};
6201
6202             $drive->{size} = $volid_hash->{$volid}->{size};
6203             my $new = print_drive($vmid, $drive);
6204             if ($new ne $conf->{$opt}) {
6205                 $changes = 1;
6206                 $conf->{$opt} = $new;
6207                 print "$prefix update disk '$opt' information.\n";
6208             }
6209         }
6210     }
6211
6212     # remove 'unusedX' entry if volume is used
6213     foreach my $opt (keys %$conf) {
6214         next if $opt !~ m/^unused\d+$/;
6215         my $volid = $conf->{$opt};
6216         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6217         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6218             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6219             $changes = 1;
6220             delete $conf->{$opt};
6221         }
6222
6223         $referenced->{$volid} = 1;
6224         $referencedpath->{$path} = 1 if $path;
6225     }
6226
6227     foreach my $volid (sort keys %$volid_hash) {
6228         next if $volid =~ m/vm-$vmid-state-/;
6229         next if $referenced->{$volid};
6230         my $path = $volid_hash->{$volid}->{path};
6231         next if !$path; # just to be sure
6232         next if $referencedpath->{$path};
6233         $changes = 1;
6234         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6235         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6236         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6237     }
6238
6239     return $changes;
6240 }
6241
6242 sub rescan {
6243     my ($vmid, $nolock, $dryrun) = @_;
6244
6245     my $cfg = PVE::Storage::config();
6246
6247     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6248     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6249     foreach my $stor (keys %{$cfg->{ids}}) {
6250         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6251     }
6252
6253     print "rescan volumes...\n";
6254     my $volid_hash = scan_volids($cfg, $vmid);
6255
6256     my $updatefn =  sub {
6257         my ($vmid) = @_;
6258
6259         my $conf = PVE::QemuConfig->load_config($vmid);
6260
6261         PVE::QemuConfig->check_lock($conf);
6262
6263         my $vm_volids = {};
6264         foreach my $volid (keys %$volid_hash) {
6265             my $info = $volid_hash->{$volid};
6266             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6267         }
6268
6269         my $changes = update_disksize($vmid, $conf, $vm_volids);
6270
6271         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6272     };
6273
6274     if (defined($vmid)) {
6275         if ($nolock) {
6276             &$updatefn($vmid);
6277         } else {
6278             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6279         }
6280     } else {
6281         my $vmlist = config_list();
6282         foreach my $vmid (keys %$vmlist) {
6283             if ($nolock) {
6284                 &$updatefn($vmid);
6285             } else {
6286                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6287             }
6288         }
6289     }
6290 }
6291
6292 sub restore_vma_archive {
6293     my ($archive, $vmid, $user, $opts, $comp) = @_;
6294
6295     my $readfrom = $archive;
6296
6297     my $cfg = PVE::Storage::config();
6298     my $commands = [];
6299     my $bwlimit = $opts->{bwlimit};
6300
6301     my $dbg_cmdstring = '';
6302     my $add_pipe = sub {
6303         my ($cmd) = @_;
6304         push @$commands, $cmd;
6305         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6306         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6307         $readfrom = '-';
6308     };
6309
6310     my $input = undef;
6311     if ($archive eq '-') {
6312         $input = '<&STDIN';
6313     } else {
6314         # If we use a backup from a PVE defined storage we also consider that
6315         # storage's rate limit:
6316         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6317         if (defined($volid)) {
6318             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6319             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6320             if ($readlimit) {
6321                 print STDERR "applying read rate limit: $readlimit\n";
6322                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6323                 $add_pipe->($cstream);
6324             }
6325         }
6326     }
6327
6328     if ($comp) {
6329         my $cmd;
6330         if ($comp eq 'gzip') {
6331             $cmd = ['zcat', $readfrom];
6332         } elsif ($comp eq 'lzop') {
6333             $cmd = ['lzop', '-d', '-c', $readfrom];
6334         } else {
6335             die "unknown compression method '$comp'\n";
6336         }
6337         $add_pipe->($cmd);
6338     }
6339
6340     my $tmpdir = "/var/tmp/vzdumptmp$$";
6341     rmtree $tmpdir;
6342
6343     # disable interrupts (always do cleanups)
6344     local $SIG{INT} =
6345         local $SIG{TERM} =
6346         local $SIG{QUIT} =
6347         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6348
6349     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6350     POSIX::mkfifo($mapfifo, 0600);
6351     my $fifofh;
6352
6353     my $openfifo = sub {
6354         open($fifofh, '>', $mapfifo) || die $!;
6355     };
6356
6357     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6358
6359     my $oldtimeout;
6360     my $timeout = 5;
6361
6362     my $devinfo = {};
6363
6364     my $rpcenv = PVE::RPCEnvironment::get();
6365
6366     my $conffile = PVE::QemuConfig->config_file($vmid);
6367     my $tmpfn = "$conffile.$$.tmp";
6368
6369     # Note: $oldconf is undef if VM does not exists
6370     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6371     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6372
6373     my %storage_limits;
6374
6375     my $print_devmap = sub {
6376         my $virtdev_hash = {};
6377
6378         my $cfgfn = "$tmpdir/qemu-server.conf";
6379
6380         # we can read the config - that is already extracted
6381         my $fh = IO::File->new($cfgfn, "r") ||
6382             "unable to read qemu-server.conf - $!\n";
6383
6384         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6385         if (-f $fwcfgfn) {
6386             my $pve_firewall_dir = '/etc/pve/firewall';
6387             mkdir $pve_firewall_dir; # make sure the dir exists
6388             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6389         }
6390
6391         while (defined(my $line = <$fh>)) {
6392             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6393                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6394                 die "archive does not contain data for drive '$virtdev'\n"
6395                     if !$devinfo->{$devname};
6396                 if (defined($opts->{storage})) {
6397                     $storeid = $opts->{storage} || 'local';
6398                 } elsif (!$storeid) {
6399                     $storeid = 'local';
6400                 }
6401                 $format = 'raw' if !$format;
6402                 $devinfo->{$devname}->{devname} = $devname;
6403                 $devinfo->{$devname}->{virtdev} = $virtdev;
6404                 $devinfo->{$devname}->{format} = $format;
6405                 $devinfo->{$devname}->{storeid} = $storeid;
6406
6407                 # check permission on storage
6408                 my $pool = $opts->{pool}; # todo: do we need that?
6409                 if ($user ne 'root@pam') {
6410                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6411                 }
6412
6413                 $storage_limits{$storeid} = $bwlimit;
6414
6415                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6416             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6417                 my $virtdev = $1;
6418                 my $drive = parse_drive($virtdev, $2);
6419                 if (drive_is_cloudinit($drive)) {
6420                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6421                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6422                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6423
6424                     my $d = {
6425                         format => $format,
6426                         storeid => $opts->{storage} // $storeid,
6427                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6428                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6429                         name => "vm-$vmid-cloudinit",
6430                         is_cloudinit => 1,
6431                     };
6432                     $virtdev_hash->{$virtdev} = $d;
6433                 }
6434             }
6435         }
6436
6437         foreach my $key (keys %storage_limits) {
6438             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6439             next if !$limit;
6440             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6441             $storage_limits{$key} = $limit * 1024;
6442         }
6443
6444         foreach my $devname (keys %$devinfo) {
6445             die "found no device mapping information for device '$devname'\n"
6446                 if !$devinfo->{$devname}->{virtdev};
6447         }
6448
6449         # create empty/temp config
6450         if ($oldconf) {
6451             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6452             foreach_drive($oldconf, sub {
6453                 my ($ds, $drive) = @_;
6454
6455                 return if drive_is_cdrom($drive, 1);
6456
6457                 my $volid = $drive->{file};
6458                 return if !$volid || $volid =~ m|^/|;
6459
6460                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6461                 return if !$path || !$owner || ($owner != $vmid);
6462
6463                 # Note: only delete disk we want to restore
6464                 # other volumes will become unused
6465                 if ($virtdev_hash->{$ds}) {
6466                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6467                     if (my $err = $@) {
6468                         warn $err;
6469                     }
6470                 }
6471             });
6472
6473             # delete vmstate files, after the restore we have no snapshots anymore
6474             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6475                 my $snap = $oldconf->{snapshots}->{$snapname};
6476                 if ($snap->{vmstate}) {
6477                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6478                     if (my $err = $@) {
6479                         warn $err;
6480                     }
6481                 }
6482             }
6483         }
6484
6485         my $map = {};
6486         foreach my $virtdev (sort keys %$virtdev_hash) {
6487             my $d = $virtdev_hash->{$virtdev};
6488             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6489             my $storeid = $d->{storeid};
6490             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6491
6492             my $map_opts = '';
6493             if (my $limit = $storage_limits{$storeid}) {
6494                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6495             }
6496
6497             # test if requested format is supported
6498             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6499             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6500             $d->{format} = $defFormat if !$supported;
6501
6502             my $name;
6503             if ($d->{is_cloudinit}) {
6504                 $name = $d->{name};
6505                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6506             }
6507
6508             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6509             print STDERR "new volume ID is '$volid'\n";
6510             $d->{volid} = $volid;
6511
6512             PVE::Storage::activate_volumes($cfg, [$volid]);
6513
6514             my $write_zeros = 1;
6515             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6516                 $write_zeros = 0;
6517             }
6518
6519             if (!$d->{is_cloudinit}) {
6520                 my $path = PVE::Storage::path($cfg, $volid);
6521
6522                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6523
6524                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6525             }
6526             $map->{$virtdev} = $volid;
6527         }
6528
6529         $fh->seek(0, 0) || die "seek failed - $!\n";
6530
6531         my $outfd = new IO::File ($tmpfn, "w") ||
6532             die "unable to write config for VM $vmid\n";
6533
6534         my $cookie = { netcount => 0 };
6535         while (defined(my $line = <$fh>)) {
6536             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6537         }
6538
6539         $fh->close();
6540         $outfd->close();
6541     };
6542
6543     eval {
6544         # enable interrupts
6545         local $SIG{INT} =
6546             local $SIG{TERM} =
6547             local $SIG{QUIT} =
6548             local $SIG{HUP} =
6549             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6550         local $SIG{ALRM} = sub { die "got timeout\n"; };
6551
6552         $oldtimeout = alarm($timeout);
6553
6554         my $parser = sub {
6555             my $line = shift;
6556
6557             print "$line\n";
6558
6559             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6560                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6561                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6562             } elsif ($line =~ m/^CTIME: /) {
6563                 # we correctly received the vma config, so we can disable
6564                 # the timeout now for disk allocation (set to 10 minutes, so
6565                 # that we always timeout if something goes wrong)
6566                 alarm(600);
6567                 &$print_devmap();
6568                 print $fifofh "done\n";
6569                 my $tmp = $oldtimeout || 0;
6570                 $oldtimeout = undef;
6571                 alarm($tmp);
6572                 close($fifofh);
6573             }
6574         };
6575
6576         print "restore vma archive: $dbg_cmdstring\n";
6577         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6578     };
6579     my $err = $@;
6580
6581     alarm($oldtimeout) if $oldtimeout;
6582
6583     my $vollist = [];
6584     foreach my $devname (keys %$devinfo) {
6585         my $volid = $devinfo->{$devname}->{volid};
6586         push @$vollist, $volid if $volid;
6587     }
6588
6589     PVE::Storage::deactivate_volumes($cfg, $vollist);
6590
6591     unlink $mapfifo;
6592
6593     if ($err) {
6594         rmtree $tmpdir;
6595         unlink $tmpfn;
6596
6597         foreach my $devname (keys %$devinfo) {
6598             my $volid = $devinfo->{$devname}->{volid};
6599             next if !$volid;
6600             eval {
6601                 if ($volid =~ m|^/|) {
6602                     unlink $volid || die 'unlink failed\n';
6603                 } else {
6604                     PVE::Storage::vdisk_free($cfg, $volid);
6605                 }
6606                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6607             };
6608             print STDERR "unable to cleanup '$volid' - $@" if $@;
6609         }
6610         die $err;
6611     }
6612
6613     rmtree $tmpdir;
6614
6615     rename($tmpfn, $conffile) ||
6616         die "unable to commit configuration file '$conffile'\n";
6617
6618     PVE::Cluster::cfs_update(); # make sure we read new file
6619
6620     eval { rescan($vmid, 1); };
6621     warn $@ if $@;
6622 }
6623
6624 sub restore_tar_archive {
6625     my ($archive, $vmid, $user, $opts) = @_;
6626
6627     if ($archive ne '-') {
6628         my $firstfile = tar_archive_read_firstfile($archive);
6629         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6630             if $firstfile ne 'qemu-server.conf';
6631     }
6632
6633     my $storecfg = PVE::Storage::config();
6634
6635     # destroy existing data - keep empty config
6636     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6637     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6638
6639     my $tocmd = "/usr/lib/qemu-server/qmextract";
6640
6641     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6642     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6643     $tocmd .= ' --prealloc' if $opts->{prealloc};
6644     $tocmd .= ' --info' if $opts->{info};
6645
6646     # tar option "xf" does not autodetect compression when read from STDIN,
6647     # so we pipe to zcat
6648     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6649         PVE::Tools::shellquote("--to-command=$tocmd");
6650
6651     my $tmpdir = "/var/tmp/vzdumptmp$$";
6652     mkpath $tmpdir;
6653
6654     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6655     local $ENV{VZDUMP_VMID} = $vmid;
6656     local $ENV{VZDUMP_USER} = $user;
6657
6658     my $conffile = PVE::QemuConfig->config_file($vmid);
6659     my $tmpfn = "$conffile.$$.tmp";
6660
6661     # disable interrupts (always do cleanups)
6662     local $SIG{INT} =
6663         local $SIG{TERM} =
6664         local $SIG{QUIT} =
6665         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6666
6667     eval {
6668         # enable interrupts
6669         local $SIG{INT} =
6670             local $SIG{TERM} =
6671             local $SIG{QUIT} =
6672             local $SIG{HUP} =
6673             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6674
6675         if ($archive eq '-') {
6676             print "extracting archive from STDIN\n";
6677             run_command($cmd, input => "<&STDIN");
6678         } else {
6679             print "extracting archive '$archive'\n";
6680             run_command($cmd);
6681         }
6682
6683         return if $opts->{info};
6684
6685         # read new mapping
6686         my $map = {};
6687         my $statfile = "$tmpdir/qmrestore.stat";
6688         if (my $fd = IO::File->new($statfile, "r")) {
6689             while (defined (my $line = <$fd>)) {
6690                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6691                     $map->{$1} = $2 if $1;
6692                 } else {
6693                     print STDERR "unable to parse line in statfile - $line\n";
6694                 }
6695             }
6696             $fd->close();
6697         }
6698
6699         my $confsrc = "$tmpdir/qemu-server.conf";
6700
6701         my $srcfd = new IO::File($confsrc, "r") ||
6702             die "unable to open file '$confsrc'\n";
6703
6704         my $outfd = new IO::File ($tmpfn, "w") ||
6705             die "unable to write config for VM $vmid\n";
6706
6707         my $cookie = { netcount => 0 };
6708         while (defined (my $line = <$srcfd>)) {
6709             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6710         }
6711
6712         $srcfd->close();
6713         $outfd->close();
6714     };
6715     my $err = $@;
6716
6717     if ($err) {
6718
6719         unlink $tmpfn;
6720
6721         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6722
6723         die $err;
6724     }
6725
6726     rmtree $tmpdir;
6727
6728     rename $tmpfn, $conffile ||
6729         die "unable to commit configuration file '$conffile'\n";
6730
6731     PVE::Cluster::cfs_update(); # make sure we read new file
6732
6733     eval { rescan($vmid, 1); };
6734     warn $@ if $@;
6735 };
6736
6737 sub foreach_storage_used_by_vm {
6738     my ($conf, $func) = @_;
6739
6740     my $sidhash = {};
6741
6742     foreach_drive($conf, sub {
6743         my ($ds, $drive) = @_;
6744         return if drive_is_cdrom($drive);
6745
6746         my $volid = $drive->{file};
6747
6748         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6749         $sidhash->{$sid} = $sid if $sid;
6750     });
6751
6752     foreach my $sid (sort keys %$sidhash) {
6753         &$func($sid);
6754     }
6755 }
6756
6757 sub do_snapshots_with_qemu {
6758     my ($storecfg, $volid) = @_;
6759
6760     my $storage_name = PVE::Storage::parse_volume_id($volid);
6761     my $scfg = $storecfg->{ids}->{$storage_name};
6762
6763     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6764         return 1;
6765     }
6766
6767     if ($volid =~ m/\.(qcow2|qed)$/){
6768         return 1;
6769     }
6770
6771     return undef;
6772 }
6773
6774 sub qga_check_running {
6775     my ($vmid, $nowarn) = @_;
6776
6777     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6778     if ($@) {
6779         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6780         return 0;
6781     }
6782     return 1;
6783 }
6784
6785 sub template_create {
6786     my ($vmid, $conf, $disk) = @_;
6787
6788     my $storecfg = PVE::Storage::config();
6789
6790     foreach_drive($conf, sub {
6791         my ($ds, $drive) = @_;
6792
6793         return if drive_is_cdrom($drive);
6794         return if $disk && $ds ne $disk;
6795
6796         my $volid = $drive->{file};
6797         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6798
6799         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6800         $drive->{file} = $voliddst;
6801         $conf->{$ds} = print_drive($vmid, $drive);
6802         PVE::QemuConfig->write_config($vmid, $conf);
6803     });
6804 }
6805
6806 sub convert_iscsi_path {
6807     my ($path) = @_;
6808
6809     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6810         my $portal = $1;
6811         my $target = $2;
6812         my $lun = $3;
6813
6814         my $initiator_name = get_initiator_name();
6815
6816         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6817                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6818     }
6819
6820     die "cannot convert iscsi path '$path', unkown format\n";
6821 }
6822
6823 sub qemu_img_convert {
6824     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6825
6826     my $storecfg = PVE::Storage::config();
6827     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6828     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6829
6830     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6831
6832     my $cachemode;
6833     my $src_path;
6834     my $src_is_iscsi = 0;
6835     my $src_format = 'raw';
6836
6837     if ($src_storeid) {
6838         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6839         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6840         $src_format = qemu_img_format($src_scfg, $src_volname);
6841         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6842         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6843         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6844     } elsif (-f $src_volid) {
6845         $src_path = $src_volid;
6846         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6847             $src_format = $1;
6848         }
6849     }
6850
6851     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6852
6853     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6854     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6855     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6856     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6857
6858     my $cmd = [];
6859     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6860     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6861     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6862     push @$cmd, '-T', $cachemode if defined($cachemode);
6863
6864     if ($src_is_iscsi) {
6865         push @$cmd, '--image-opts';
6866         $src_path = convert_iscsi_path($src_path);
6867     } else {
6868         push @$cmd, '-f', $src_format;
6869     }
6870
6871     if ($dst_is_iscsi) {
6872         push @$cmd, '--target-image-opts';
6873         $dst_path = convert_iscsi_path($dst_path);
6874     } else {
6875         push @$cmd, '-O', $dst_format;
6876     }
6877
6878     push @$cmd, $src_path;
6879
6880     if (!$dst_is_iscsi && $is_zero_initialized) {
6881         push @$cmd, "zeroinit:$dst_path";
6882     } else {
6883         push @$cmd, $dst_path;
6884     }
6885
6886     my $parser = sub {
6887         my $line = shift;
6888         if($line =~ m/\((\S+)\/100\%\)/){
6889             my $percent = $1;
6890             my $transferred = int($size * $percent / 100);
6891             my $remaining = $size - $transferred;
6892
6893             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6894         }
6895
6896     };
6897
6898     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6899     my $err = $@;
6900     die "copy failed: $err" if $err;
6901 }
6902
6903 sub qemu_img_format {
6904     my ($scfg, $volname) = @_;
6905
6906     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6907         return $1;
6908     } else {
6909         return "raw";
6910     }
6911 }
6912
6913 sub qemu_drive_mirror {
6914     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6915
6916     $jobs = {} if !$jobs;
6917
6918     my $qemu_target;
6919     my $format;
6920     $jobs->{"drive-$drive"} = {};
6921
6922     if ($dst_volid =~ /^nbd:/) {
6923         $qemu_target = $dst_volid;
6924         $format = "nbd";
6925     } else {
6926         my $storecfg = PVE::Storage::config();
6927         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6928
6929         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6930
6931         $format = qemu_img_format($dst_scfg, $dst_volname);
6932
6933         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6934
6935         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6936     }
6937
6938     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6939     $opts->{format} = $format if $format;
6940
6941     if (defined($bwlimit)) {
6942         $opts->{speed} = $bwlimit * 1024;
6943         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6944     } else {
6945         print "drive mirror is starting for drive-$drive\n";
6946     }
6947
6948     # if a job already runs for this device we get an error, catch it for cleanup
6949     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6950     if (my $err = $@) {
6951         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6952         warn "$@\n" if $@;
6953         die "mirroring error: $err\n";
6954     }
6955
6956     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6957 }
6958
6959 sub qemu_drive_mirror_monitor {
6960     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6961
6962     eval {
6963         my $err_complete = 0;
6964
6965         while (1) {
6966             die "storage migration timed out\n" if $err_complete > 300;
6967
6968             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6969
6970             my $running_mirror_jobs = {};
6971             foreach my $stat (@$stats) {
6972                 next if $stat->{type} ne 'mirror';
6973                 $running_mirror_jobs->{$stat->{device}} = $stat;
6974             }
6975
6976             my $readycounter = 0;
6977
6978             foreach my $job (keys %$jobs) {
6979
6980                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6981                     print "$job : finished\n";
6982                     delete $jobs->{$job};
6983                     next;
6984                 }
6985
6986                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6987
6988                 my $busy = $running_mirror_jobs->{$job}->{busy};
6989                 my $ready = $running_mirror_jobs->{$job}->{ready};
6990                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6991                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6992                     my $remaining = $total - $transferred;
6993                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6994
6995                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6996                 }
6997
6998                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6999             }
7000
7001             last if scalar(keys %$jobs) == 0;
7002
7003             if ($readycounter == scalar(keys %$jobs)) {
7004                 print "all mirroring jobs are ready \n";
7005                 last if $skipcomplete; #do the complete later
7006
7007                 if ($vmiddst && $vmiddst != $vmid) {
7008                     my $agent_running = $qga && qga_check_running($vmid);
7009                     if ($agent_running) {
7010                         print "freeze filesystem\n";
7011                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7012                     } else {
7013                         print "suspend vm\n";
7014                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7015                     }
7016
7017                     # if we clone a disk for a new target vm, we don't switch the disk
7018                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7019
7020                     if ($agent_running) {
7021                         print "unfreeze filesystem\n";
7022                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7023                     } else {
7024                         print "resume vm\n";
7025                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7026                     }
7027
7028                     last;
7029                 } else {
7030
7031                     foreach my $job (keys %$jobs) {
7032                         # try to switch the disk if source and destination are on the same guest
7033                         print "$job: Completing block job...\n";
7034
7035                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7036                         if ($@ =~ m/cannot be completed/) {
7037                             print "$job: Block job cannot be completed, try again.\n";
7038                             $err_complete++;
7039                         }else {
7040                             print "$job: Completed successfully.\n";
7041                             $jobs->{$job}->{complete} = 1;
7042                         }
7043                     }
7044                 }
7045             }
7046             sleep 1;
7047         }
7048     };
7049     my $err = $@;
7050
7051     if ($err) {
7052         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7053         die "mirroring error: $err";
7054     }
7055
7056 }
7057
7058 sub qemu_blockjobs_cancel {
7059     my ($vmid, $jobs) = @_;
7060
7061     foreach my $job (keys %$jobs) {
7062         print "$job: Cancelling block job\n";
7063         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7064         $jobs->{$job}->{cancel} = 1;
7065     }
7066
7067     while (1) {
7068         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7069
7070         my $running_jobs = {};
7071         foreach my $stat (@$stats) {
7072             $running_jobs->{$stat->{device}} = $stat;
7073         }
7074
7075         foreach my $job (keys %$jobs) {
7076
7077             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7078                 print "$job: Done.\n";
7079                 delete $jobs->{$job};
7080             }
7081         }
7082
7083         last if scalar(keys %$jobs) == 0;
7084
7085         sleep 1;
7086     }
7087 }
7088
7089 sub clone_disk {
7090     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7091         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7092
7093     my $newvolid;
7094
7095     if (!$full) {
7096         print "create linked clone of drive $drivename ($drive->{file})\n";
7097         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7098         push @$newvollist, $newvolid;
7099     } else {
7100
7101         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7102         $storeid = $storage if $storage;
7103
7104         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7105         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7106
7107         print "create full clone of drive $drivename ($drive->{file})\n";
7108         my $name = undef;
7109         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7110         push @$newvollist, $newvolid;
7111
7112         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7113
7114         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7115         if (!$running || $snapname) {
7116             # TODO: handle bwlimits
7117             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7118         } else {
7119
7120             my $kvmver = get_running_qemu_version ($vmid);
7121             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7122                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7123                     if $drive->{iothread};
7124             }
7125
7126             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7127         }
7128     }
7129
7130     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7131
7132     my $disk = $drive;
7133     $disk->{format} = undef;
7134     $disk->{file} = $newvolid;
7135     $disk->{size} = $size;
7136
7137     return $disk;
7138 }
7139
7140 # this only works if VM is running
7141 sub get_current_qemu_machine {
7142     my ($vmid) = @_;
7143
7144     my $cmd = { execute => 'query-machines', arguments => {} };
7145     my $res = vm_qmp_command($vmid, $cmd);
7146
7147     my ($current, $default);
7148     foreach my $e (@$res) {
7149         $default = $e->{name} if $e->{'is-default'};
7150         $current = $e->{name} if $e->{'is-current'};
7151     }
7152
7153     # fallback to the default machine if current is not supported by qemu
7154     return $current || $default || 'pc';
7155 }
7156
7157 sub get_running_qemu_version {
7158     my ($vmid) = @_;
7159     my $cmd = { execute => 'query-version', arguments => {} };
7160     my $res = vm_qmp_command($vmid, $cmd);
7161     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7162 }
7163
7164 sub qemu_machine_feature_enabled {
7165     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7166
7167     my $current_major;
7168     my $current_minor;
7169
7170     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7171
7172         $current_major = $3;
7173         $current_minor = $4;
7174
7175     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7176
7177         $current_major = $1;
7178         $current_minor = $2;
7179     }
7180
7181     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7182 }
7183
7184 # gets in pairs the versions you want to compares, i.e.:
7185 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7186 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7187 sub version_cmp {
7188     my @versions = @_;
7189
7190     my $size = scalar(@versions);
7191
7192     return 0 if $size == 0;
7193     die "cannot compare odd count of versions" if $size & 1;
7194
7195     for (my $i = 0; $i < $size; $i += 2) {
7196         my ($a, $b) = splice(@versions, 0, 2);
7197         $a //= 0;
7198         $b //= 0;
7199
7200         return 1 if $a > $b;
7201         return -1 if $a < $b;
7202     }
7203     return 0;
7204 }
7205
7206 sub runs_at_least_qemu_version {
7207     my ($vmid, $major, $minor, $extra) = @_;
7208
7209     my $v = PVE::QemuServer::vm_qmp_command($vmid, { execute => 'query-version' });
7210     $v = $v->{qemu};
7211
7212     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7213 }
7214
7215 sub qemu_machine_pxe {
7216     my ($vmid, $conf) = @_;
7217
7218     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7219
7220     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7221         $machine .= '.pxe';
7222     }
7223
7224     return $machine;
7225 }
7226
7227 sub qemu_use_old_bios_files {
7228     my ($machine_type) = @_;
7229
7230     return if !$machine_type;
7231
7232     my $use_old_bios_files = undef;
7233
7234     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7235         $machine_type = $1;
7236         $use_old_bios_files = 1;
7237     } else {
7238         my $kvmver = kvm_user_version();
7239         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7240         # load new efi bios files on migration. So this hack is required to allow
7241         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7242         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7243         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7244     }
7245
7246     return ($use_old_bios_files, $machine_type);
7247 }
7248
7249 sub create_efidisk($$$$$) {
7250     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7251
7252     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7253     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7254
7255     my $vars_size_b = -s $ovmf_vars;
7256     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7257     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7258     PVE::Storage::activate_volumes($storecfg, [$volid]);
7259
7260     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7261
7262     return ($volid, $vars_size);
7263 }
7264
7265 sub vm_iothreads_list {
7266     my ($vmid) = @_;
7267
7268     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7269
7270     my $iothreads = {};
7271     foreach my $iothread (@$res) {
7272         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7273     }
7274
7275     return $iothreads;
7276 }
7277
7278 sub scsihw_infos {
7279     my ($conf, $drive) = @_;
7280
7281     my $maxdev = 0;
7282
7283     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7284         $maxdev = 7;
7285     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7286         $maxdev = 1;
7287     } else {
7288         $maxdev = 256;
7289     }
7290
7291     my $controller = int($drive->{index} / $maxdev);
7292     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7293
7294     return ($maxdev, $controller, $controller_prefix);
7295 }
7296
7297 sub add_hyperv_enlightenments {
7298     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7299
7300     return if $winversion < 6;
7301     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7302
7303     if ($gpu_passthrough || defined($hv_vendor_id)) {
7304         $hv_vendor_id //= 'proxmox';
7305         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7306     }
7307
7308     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7309         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7310         push @$cpuFlags , 'hv_vapic';
7311         push @$cpuFlags , 'hv_time';
7312     } else {
7313         push @$cpuFlags , 'hv_spinlocks=0xffff';
7314     }
7315
7316     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7317         push @$cpuFlags , 'hv_reset';
7318         push @$cpuFlags , 'hv_vpindex';
7319         push @$cpuFlags , 'hv_runtime';
7320     }
7321
7322     if ($winversion >= 7) {
7323         push @$cpuFlags , 'hv_relaxed';
7324
7325         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7326             push @$cpuFlags , 'hv_synic';
7327             push @$cpuFlags , 'hv_stimer';
7328         }
7329
7330         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7331             push @$cpuFlags , 'hv_ipi';
7332         }
7333     }
7334 }
7335
7336 sub windows_version {
7337     my ($ostype) = @_;
7338
7339     return 0 if !$ostype;
7340
7341     my $winversion = 0;
7342
7343     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7344         $winversion = 5;
7345     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7346         $winversion = 6;
7347     } elsif ($ostype =~ m/^win(\d+)$/) {
7348         $winversion = $1;
7349     }
7350
7351     return $winversion;
7352 }
7353
7354 sub resolve_dst_disk_format {
7355         my ($storecfg, $storeid, $src_volname, $format) = @_;
7356         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7357
7358         if (!$format) {
7359             # if no target format is specified, use the source disk format as hint
7360             if ($src_volname) {
7361                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7362                 $format = qemu_img_format($scfg, $src_volname);
7363             } else {
7364                 return $defFormat;
7365             }
7366         }
7367
7368         # test if requested format is supported - else use default
7369         my $supported = grep { $_ eq $format } @$validFormats;
7370         $format = $defFormat if !$supported;
7371         return $format;
7372 }
7373
7374 sub resolve_first_disk {
7375     my $conf = shift;
7376     my @disks = PVE::QemuServer::valid_drive_names();
7377     my $firstdisk;
7378     foreach my $ds (reverse @disks) {
7379         next if !$conf->{$ds};
7380         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7381         next if PVE::QemuServer::drive_is_cdrom($disk);
7382         $firstdisk = $ds;
7383     }
7384     return $firstdisk;
7385 }
7386
7387 sub generate_uuid {
7388     my ($uuid, $uuid_str);
7389     UUID::generate($uuid);
7390     UUID::unparse($uuid, $uuid_str);
7391     return $uuid_str;
7392 }
7393
7394 sub generate_smbios1_uuid {
7395     return "uuid=".generate_uuid();
7396 }
7397
7398 sub nbd_stop {
7399     my ($vmid) = @_;
7400
7401     vm_mon_cmd($vmid, 'nbd-server-stop');
7402 }
7403
7404 sub create_reboot_request {
7405     my ($vmid) = @_;
7406     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7407         or die "failed to create reboot trigger file: $!\n";
7408     close($fh);
7409 }
7410
7411 sub clear_reboot_request {
7412     my ($vmid) = @_;
7413     my $path = "/run/qemu-server/$vmid.reboot";
7414     my $res = 0;
7415
7416     $res = unlink($path);
7417     die "could not remove reboot request for $vmid: $!"
7418         if !$res && $! != POSIX::ENOENT;
7419
7420     return $res;
7421 }
7422
7423 # bash completion helper
7424
7425 sub complete_backup_archives {
7426     my ($cmdname, $pname, $cvalue) = @_;
7427
7428     my $cfg = PVE::Storage::config();
7429
7430     my $storeid;
7431
7432     if ($cvalue =~ m/^([^:]+):/) {
7433         $storeid = $1;
7434     }
7435
7436     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7437
7438     my $res = [];
7439     foreach my $id (keys %$data) {
7440         foreach my $item (@{$data->{$id}}) {
7441             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7442             push @$res, $item->{volid} if defined($item->{volid});
7443         }
7444     }
7445
7446     return $res;
7447 }
7448
7449 my $complete_vmid_full = sub {
7450     my ($running) = @_;
7451
7452     my $idlist = vmstatus();
7453
7454     my $res = [];
7455
7456     foreach my $id (keys %$idlist) {
7457         my $d = $idlist->{$id};
7458         if (defined($running)) {
7459             next if $d->{template};
7460             next if $running && $d->{status} ne 'running';
7461             next if !$running && $d->{status} eq 'running';
7462         }
7463         push @$res, $id;
7464
7465     }
7466     return $res;
7467 };
7468
7469 sub complete_vmid {
7470     return &$complete_vmid_full();
7471 }
7472
7473 sub complete_vmid_stopped {
7474     return &$complete_vmid_full(0);
7475 }
7476
7477 sub complete_vmid_running {
7478     return &$complete_vmid_full(1);
7479 }
7480
7481 sub complete_storage {
7482
7483     my $cfg = PVE::Storage::config();
7484     my $ids = $cfg->{ids};
7485
7486     my $res = [];
7487     foreach my $sid (keys %$ids) {
7488         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7489         next if !$ids->{$sid}->{content}->{images};
7490         push @$res, $sid;
7491     }
7492
7493     return $res;
7494 }
7495
7496 1;