PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27
  28 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  29 use PVE::Exception qw(raise raise_param_exc);
  30 use PVE::GuestHelpers;
  31 use PVE::INotify;
  32 use PVE::JSONSchema qw(get_standard_option);
  33 use PVE::ProcFSTools;
  34 use PVE::RPCEnvironment;
  35 use PVE::SafeSyslog;
  36 use PVE::Storage;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  40
  41 use PVE::QMPClient;
  42 use PVE::QemuConfig;
  43 use PVE::QemuServer::Cloudinit;
  44 use PVE::QemuServer::Memory;
  45 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  46 use PVE::QemuServer::USB qw(parse_usb_device);
  47
  48 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  49 my $OVMF = {
  50     x86_64 => [
  51         "$EDK2_FW_BASE/OVMF_CODE.fd",
  52         "$EDK2_FW_BASE/OVMF_VARS.fd"
  53     ],
  54     aarch64 => [
  55         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  56         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  57     ],
  58 };
  59
  60 my $qemu_snap_storage = { rbd => 1 };
  61
  62 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  63
  64 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  65
  66 # Note about locking: we use flock on the config file protect
  67 # against concurent actions.
  68 # Aditionaly, we have a 'lock' setting in the config file. This
  69 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  70 # allowed when such lock is set. But you can ignore this kind of
  71 # lock with the --skiplock flag.
  72
  73 cfs_register_file('/qemu-server/',
  74                   \&parse_vm_config,
  75                   \&write_vm_config);
  76
  77 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  78     description => "Some command save/restore state from this location.",
  79     type => 'string',
  80     maxLength => 128,
  81     optional => 1,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $cpu_vendor_list = {
 122     # Intel CPUs
 123     486 => 'GenuineIntel',
 124     pentium => 'GenuineIntel',
 125     pentium2  => 'GenuineIntel',
 126     pentium3  => 'GenuineIntel',
 127     coreduo => 'GenuineIntel',
 128     core2duo => 'GenuineIntel',
 129     Conroe  => 'GenuineIntel',
 130     Penryn  => 'GenuineIntel',
 131     Nehalem  => 'GenuineIntel',
 132     'Nehalem-IBRS'  => 'GenuineIntel',
 133     Westmere => 'GenuineIntel',
 134     'Westmere-IBRS' => 'GenuineIntel',
 135     SandyBridge => 'GenuineIntel',
 136     'SandyBridge-IBRS' => 'GenuineIntel',
 137     IvyBridge => 'GenuineIntel',
 138     'IvyBridge-IBRS' => 'GenuineIntel',
 139     Haswell => 'GenuineIntel',
 140     'Haswell-IBRS' => 'GenuineIntel',
 141     'Haswell-noTSX' => 'GenuineIntel',
 142     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 143     Broadwell => 'GenuineIntel',
 144     'Broadwell-IBRS' => 'GenuineIntel',
 145     'Broadwell-noTSX' => 'GenuineIntel',
 146     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 147     'Skylake-Client' => 'GenuineIntel',
 148     'Skylake-Client-IBRS' => 'GenuineIntel',
 149     'Skylake-Server' => 'GenuineIntel',
 150     'Skylake-Server-IBRS' => 'GenuineIntel',
 151     'Cascadelake-Server' => 'GenuineIntel',
 152     KnightsMill => 'GenuineIntel',
 153
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my @supported_cpu_flags = (
 176     'pcid',
 177     'spec-ctrl',
 178     'ibpb',
 179     'ssbd',
 180     'virt-ssbd',
 181     'amd-ssbd',
 182     'amd-no-ssb',
 183     'pdpe1gb',
 184     'md-clear',
 185     'hv-tlbflush',
 186     'hv-evmcs',
 187     'aes'
 188 );
 189 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 190
 191 my $cpu_fmt = {
 192     cputype => {
 193         description => "Emulated CPU type.",
 194         type => 'string',
 195         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 196         default => 'kvm64',
 197         default_key => 1,
 198     },
 199     hidden => {
 200         description => "Do not identify as a KVM virtual machine.",
 201         type => 'boolean',
 202         optional => 1,
 203         default => 0
 204     },
 205     'hv-vendor-id' => {
 206         type => 'string',
 207         pattern => qr/[a-zA-Z0-9]{1,12}/,
 208         format_description => 'vendor-id',
 209         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 210         optional => 1,
 211     },
 212     flags => {
 213         description => "List of additional CPU flags separated by ';'."
 214                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 215                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 216         format_description => '+FLAG[;-FLAG...]',
 217         type => 'string',
 218         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 219         optional => 1,
 220     },
 221 };
 222
 223 my $watchdog_fmt = {
 224     model => {
 225         default_key => 1,
 226         type => 'string',
 227         enum => [qw(i6300esb ib700)],
 228         description => "Watchdog type to emulate.",
 229         default => 'i6300esb',
 230         optional => 1,
 231     },
 232     action => {
 233         type => 'string',
 234         enum => [qw(reset shutdown poweroff pause debug none)],
 235         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 236         optional => 1,
 237     },
 238 };
 239 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 240
 241 my $agent_fmt = {
 242     enabled => {
 243         description => "Enable/disable Qemu GuestAgent.",
 244         type => 'boolean',
 245         default => 0,
 246         default_key => 1,
 247     },
 248     fstrim_cloned_disks => {
 249         description => "Run fstrim after cloning/moving a disk.",
 250         type => 'boolean',
 251         optional => 1,
 252         default => 0
 253     },
 254 };
 255
 256 my $vga_fmt = {
 257     type => {
 258         description => "Select the VGA type.",
 259         type => 'string',
 260         default => 'std',
 261         optional => 1,
 262         default_key => 1,
 263         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 264     },
 265     memory => {
 266         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 267         type => 'integer',
 268         optional => 1,
 269         minimum => 4,
 270         maximum => 512,
 271     },
 272 };
 273
 274 my $ivshmem_fmt = {
 275     size => {
 276         type => 'integer',
 277         minimum => 1,
 278         description => "The size of the file in MB.",
 279     },
 280     name => {
 281         type => 'string',
 282         pattern => '[a-zA-Z0-9\-]+',
 283         optional => 1,
 284         format_description => 'string',
 285         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 286     },
 287 };
 288
 289 my $audio_fmt = {
 290     device => {
 291         type => 'string',
 292         enum => [qw(ich9-intel-hda intel-hda AC97)],
 293         description =>  "Configure an audio device."
 294     },
 295     driver =>  {
 296         type => 'string',
 297         enum => ['spice'],
 298         default => 'spice',
 299         optional => 1,
 300         description => "Driver backend for the audio device."
 301     },
 302 };
 303
 304 my $spice_enhancements_fmt = {
 305     foldersharing => {
 306         type => 'boolean',
 307         optional => 1,
 308         default => '0',
 309         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 310     },
 311     videostreaming =>  {
 312         type => 'string',
 313         enum => ['off', 'all', 'filter'],
 314         default => 'off',
 315         optional => 1,
 316         description => "Enable video streaming. Uses compression for detected video streams."
 317     },
 318 };
 319
 320 my $confdesc = {
 321     onboot => {
 322         optional => 1,
 323         type => 'boolean',
 324         description => "Specifies whether a VM will be started during system bootup.",
 325         default => 0,
 326     },
 327     autostart => {
 328         optional => 1,
 329         type => 'boolean',
 330         description => "Automatic restart after crash (currently ignored).",
 331         default => 0,
 332     },
 333     hotplug => {
 334         optional => 1,
 335         type => 'string', format => 'pve-hotplug-features',
 336         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 337         default => 'network,disk,usb',
 338     },
 339     reboot => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 343         default => 1,
 344     },
 345     lock => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Lock/unlock the VM.",
 349         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 350     },
 351     cpulimit => {
 352         optional => 1,
 353         type => 'number',
 354         description => "Limit of CPU usage.",
 355         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 356         minimum => 0,
 357         maximum => 128,
 358         default => 0,
 359     },
 360     cpuunits => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "CPU weight for a VM.",
 364         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 365         minimum => 2,
 366         maximum => 262144,
 367         default => 1024,
 368     },
 369     memory => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 373         minimum => 16,
 374         default => 512,
 375     },
 376     balloon => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 380         minimum => 0,
 381     },
 382     shares => {
 383         optional => 1,
 384         type => 'integer',
 385         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 386         minimum => 0,
 387         maximum => 50000,
 388         default => 1000,
 389     },
 390     keyboard => {
 391         optional => 1,
 392         type => 'string',
 393         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 394                        "It should not be necessary to set it.",
 395         enum => PVE::Tools::kvmkeymaplist(),
 396         default => undef,
 397     },
 398     name => {
 399         optional => 1,
 400         type => 'string', format => 'dns-name',
 401         description => "Set a name for the VM. Only used on the configuration web interface.",
 402     },
 403     scsihw => {
 404         optional => 1,
 405         type => 'string',
 406         description => "SCSI controller model",
 407         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 408         default => 'lsi',
 409     },
 410     description => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 414     },
 415     ostype => {
 416         optional => 1,
 417         type => 'string',
 418         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 419         description => "Specify guest operating system.",
 420         verbose_description => <<EODESC,
 421 Specify guest operating system. This is used to enable special
 422 optimization/features for specific operating systems:
 423
 424 [horizontal]
 425 other;; unspecified OS
 426 wxp;; Microsoft Windows XP
 427 w2k;; Microsoft Windows 2000
 428 w2k3;; Microsoft Windows 2003
 429 w2k8;; Microsoft Windows 2008
 430 wvista;; Microsoft Windows Vista
 431 win7;; Microsoft Windows 7
 432 win8;; Microsoft Windows 8/2012/2012r2
 433 win10;; Microsoft Windows 10/2016
 434 l24;; Linux 2.4 Kernel
 435 l26;; Linux 2.6/3.X Kernel
 436 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 437 EODESC
 438     },
 439     boot => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 443         pattern => '[acdn]{1,4}',
 444         default => 'cdn',
 445     },
 446     bootdisk => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-bootdisk',
 449         description => "Enable booting from specified disk.",
 450         pattern => '(ide|sata|scsi|virtio)\d+',
 451     },
 452     smp => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPUs. Please use option -sockets instead.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     sockets => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of CPU sockets.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     cores => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "The number of cores per socket.",
 470         minimum => 1,
 471         default => 1,
 472     },
 473     numa => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable NUMA.",
 477         default => 0,
 478     },
 479     hugepages => {
 480         optional => 1,
 481         type => 'string',
 482         description => "Enable/disable hugepages memory.",
 483         enum => [qw(any 2 1024)],
 484     },
 485     vcpus => {
 486         optional => 1,
 487         type => 'integer',
 488         description => "Number of hotplugged vcpus.",
 489         minimum => 1,
 490         default => 0,
 491     },
 492     acpi => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable ACPI.",
 496         default => 1,
 497     },
 498     agent => {
 499         optional => 1,
 500         description => "Enable/disable Qemu GuestAgent and its properties.",
 501         type => 'string',
 502         format => $agent_fmt,
 503     },
 504     kvm => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Enable/disable KVM hardware virtualization.",
 508         default => 1,
 509     },
 510     tdf => {
 511         optional => 1,
 512         type => 'boolean',
 513         description => "Enable/disable time drift fix.",
 514         default => 0,
 515     },
 516     localtime => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 520     },
 521     freeze => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 525     },
 526     vga => {
 527         optional => 1,
 528         type => 'string', format => $vga_fmt,
 529         description => "Configure the VGA hardware.",
 530         verbose_description => "Configure the VGA Hardware. If you want to use ".
 531             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 532             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 533             "is 'std' for all OS types besides some Windows versions (XP and " .
 534             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 535             "display server. For win* OS you can select how many independent " .
 536             "displays you want, Linux guests can add displays them self.\n".
 537             "You can also run without any graphic card, using a serial device as terminal.",
 538     },
 539     watchdog => {
 540         optional => 1,
 541         type => 'string', format => 'pve-qm-watchdog',
 542         description => "Create a virtual hardware watchdog device.",
 543         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 544             " (by a guest action), the watchdog must be periodically polled " .
 545             "by an agent inside the guest or else the watchdog will reset " .
 546             "the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 553         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 554         default => 'now',
 555     },
 556     startup =>  get_standard_option('pve-startup-order'),
 557     template => {
 558         optional => 1,
 559         type => 'boolean',
 560         description => "Enable/disable Template.",
 561         default => 0,
 562     },
 563     args => {
 564         optional => 1,
 565         type => 'string',
 566         description => "Arbitrary arguments passed to kvm.",
 567         verbose_description => <<EODESCR,
 568 Arbitrary arguments passed to kvm, for example:
 569
 570 args: -no-reboot -no-hpet
 571
 572 NOTE: this option is for experts only.
 573 EODESCR
 574     },
 575     tablet => {
 576         optional => 1,
 577         type => 'boolean',
 578         default => 1,
 579         description => "Enable/disable the USB tablet device.",
 580         verbose_description => "Enable/disable the USB tablet device. This device is " .
 581             "usually needed to allow absolute mouse positioning with VNC. " .
 582             "Else the mouse runs out of sync with normal VNC clients. " .
 583             "If you're running lots of console-only guests on one host, " .
 584             "you may consider disabling this to save some context switches. " .
 585             "This is turned off by default if you use spice (-vga=qxl).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => $cpu_fmt,
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 634     }),
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 652         default => 0,
 653     },
 654     bios => {
 655         optional => 1,
 656         type => 'string',
 657         enum => [ qw(seabios ovmf) ],
 658         description => "Select BIOS implementation.",
 659         default => 'seabios',
 660     },
 661     vmgenid => {
 662         type => 'string',
 663         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 664         format_description => 'UUID',
 665         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 666         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 667             " 128-bit integer value identifier to the guest OS. This allows to".
 668             " notify the guest operating system when the virtual machine is".
 669             " executed with a different configuration (e.g. snapshot execution".
 670             " or creation from a template). The guest operating system notices".
 671             " the change, and is then able to react as appropriate by marking".
 672             " its copies of distributed databases as dirty, re-initializing its".
 673             " random number generator, etc.\n".
 674             "Note that auto-creation only works when done throug API/CLI create".
 675             " or update methods, but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 689         optional => 1,
 690     },
 691     audio0 => {
 692         type => 'string',
 693         format => $audio_fmt,
 694         description => "Configure a audio device, useful in combination with QXL/Spice.",
 695         optional => 1
 696     },
 697     spice_enhancements => {
 698         type => 'string',
 699         format => $spice_enhancements_fmt,
 700         description => "Configure additional enhancements for SPICE.",
 701         optional => 1
 702     },
 703 };
 704
 705 my $cicustom_fmt = {
 706     meta => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     network => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727 };
 728 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 729
 730 my $confdesc_cloudinit = {
 731     citype => {
 732         optional => 1,
 733         type => 'string',
 734         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 735         enum => ['configdrive2', 'nocloud'],
 736     },
 737     ciuser => {
 738         optional => 1,
 739         type => 'string',
 740         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 741     },
 742     cipassword => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 746     },
 747     cicustom => {
 748         optional => 1,
 749         type => 'string',
 750         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 751         format => 'pve-qm-cicustom',
 752     },
 753     searchdomain => {
 754         optional => 1,
 755         type => 'string',
 756         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     nameserver => {
 759         optional => 1,
 760         type => 'string', format => 'address-list',
 761         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 762     },
 763     sshkeys => {
 764         optional => 1,
 765         type => 'string',
 766         format => 'urlencoded',
 767         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 768     },
 769 };
 770
 771 # what about other qemu settings ?
 772 #cpu => 'string',
 773 #machine => 'string',
 774 #fda => 'file',
 775 #fdb => 'file',
 776 #mtdblock => 'file',
 777 #sd => 'file',
 778 #pflash => 'file',
 779 #snapshot => 'bool',
 780 #bootp => 'file',
 781 ##tftp => 'dir',
 782 ##smb => 'dir',
 783 #kernel => 'file',
 784 #append => 'string',
 785 #initrd => 'file',
 786 ##soundhw => 'string',
 787
 788 while (my ($k, $v) = each %$confdesc) {
 789     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 790 }
 791
 792 my $MAX_IDE_DISKS = 4;
 793 my $MAX_SCSI_DISKS = 14;
 794 my $MAX_VIRTIO_DISKS = 16;
 795 my $MAX_SATA_DISKS = 6;
 796 my $MAX_USB_DEVICES = 5;
 797 my $MAX_NETS = 32;
 798 my $MAX_UNUSED_DISKS = 256;
 799 my $MAX_HOSTPCI_DEVICES = 16;
 800 my $MAX_SERIAL_PORTS = 4;
 801 my $MAX_PARALLEL_PORTS = 3;
 802 my $MAX_NUMA = 8;
 803
 804 my $numa_fmt = {
 805     cpus => {
 806         type => "string",
 807         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 808         description => "CPUs accessing this NUMA node.",
 809         format_description => "id[-id];...",
 810     },
 811     memory => {
 812         type => "number",
 813         description => "Amount of memory this NUMA node provides.",
 814         optional => 1,
 815     },
 816     hostnodes => {
 817         type => "string",
 818         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 819         description => "Host NUMA nodes to use.",
 820         format_description => "id[-id];...",
 821         optional => 1,
 822     },
 823     policy => {
 824         type => 'string',
 825         enum => [qw(preferred bind interleave)],
 826         description => "NUMA allocation policy.",
 827         optional => 1,
 828     },
 829 };
 830 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 831 my $numadesc = {
 832     optional => 1,
 833     type => 'string', format => $numa_fmt,
 834     description => "NUMA topology.",
 835 };
 836 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 837
 838 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 839     $confdesc->{"numa$i"} = $numadesc;
 840 }
 841
 842 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 843                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 844                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 845 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 846
 847 my $net_fmt_bridge_descr = <<__EOD__;
 848 Bridge to attach the network device to. The Proxmox VE standard bridge
 849 is called 'vmbr0'.
 850
 851 If you do not specify a bridge, we create a kvm user (NATed) network
 852 device, which provides DHCP and DNS services. The following addresses
 853 are used:
 854
 855  10.0.2.2   Gateway
 856  10.0.2.3   DNS Server
 857  10.0.2.4   SMB Server
 858
 859 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 860 __EOD__
 861
 862 my $net_fmt = {
 863     macaddr  => get_standard_option('mac-addr', {
 864         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 865     }),
 866     model => {
 867         type => 'string',
 868         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 869         enum => $nic_model_list,
 870         default_key => 1,
 871     },
 872     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 873     bridge => {
 874         type => 'string',
 875         description => $net_fmt_bridge_descr,
 876         format_description => 'bridge',
 877         optional => 1,
 878     },
 879     queues => {
 880         type => 'integer',
 881         minimum => 0, maximum => 16,
 882         description => 'Number of packet queues to be used on the device.',
 883         optional => 1,
 884     },
 885     rate => {
 886         type => 'number',
 887         minimum => 0,
 888         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 889         optional => 1,
 890     },
 891     tag => {
 892         type => 'integer',
 893         minimum => 1, maximum => 4094,
 894         description => 'VLAN tag to apply to packets on this interface.',
 895         optional => 1,
 896     },
 897     trunks => {
 898         type => 'string',
 899         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 900         description => 'VLAN trunks to pass through this interface.',
 901         format_description => 'vlanid[;vlanid...]',
 902         optional => 1,
 903     },
 904     firewall => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be protected by the firewall.',
 907         optional => 1,
 908     },
 909     link_down => {
 910         type => 'boolean',
 911         description => 'Whether this interface should be disconnected (like pulling the plug).',
 912         optional => 1,
 913     },
 914 };
 915
 916 my $netdesc = {
 917     optional => 1,
 918     type => 'string', format => $net_fmt,
 919     description => "Specify network devices.",
 920 };
 921
 922 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 923
 924 my $ipconfig_fmt = {
 925     ip => {
 926         type => 'string',
 927         format => 'pve-ipv4-config',
 928         format_description => 'IPv4Format/CIDR',
 929         description => 'IPv4 address in CIDR format.',
 930         optional => 1,
 931         default => 'dhcp',
 932     },
 933     gw => {
 934         type => 'string',
 935         format => 'ipv4',
 936         format_description => 'GatewayIPv4',
 937         description => 'Default gateway for IPv4 traffic.',
 938         optional => 1,
 939         requires => 'ip',
 940     },
 941     ip6 => {
 942         type => 'string',
 943         format => 'pve-ipv6-config',
 944         format_description => 'IPv6Format/CIDR',
 945         description => 'IPv6 address in CIDR format.',
 946         optional => 1,
 947         default => 'dhcp',
 948     },
 949     gw6 => {
 950         type => 'string',
 951         format => 'ipv6',
 952         format_description => 'GatewayIPv6',
 953         description => 'Default gateway for IPv6 traffic.',
 954         optional => 1,
 955         requires => 'ip6',
 956     },
 957 };
 958 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 959 my $ipconfigdesc = {
 960     optional => 1,
 961     type => 'string', format => 'pve-qm-ipconfig',
 962     description => <<'EODESCR',
 963 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 964
 965 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 966
 967 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 968 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 969
 970 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return undef if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $drivename_hash;
1002
1003 my %drivedesc_base = (
1004     volume => { alias => 'file' },
1005     file => {
1006         type => 'string',
1007         format => 'pve-volume-id-or-qm-path',
1008         default_key => 1,
1009         format_description => 'volume',
1010         description => "The drive's backing volume.",
1011     },
1012     media => {
1013         type => 'string',
1014         enum => [qw(cdrom disk)],
1015         description => "The drive's media type.",
1016         default => 'disk',
1017         optional => 1
1018     },
1019     cyls => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific cylinder count.",
1022         optional => 1
1023     },
1024     heads => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific head count.",
1027         optional => 1
1028     },
1029     secs => {
1030         type => 'integer',
1031         description => "Force the drive's physical geometry to have a specific sector count.",
1032         optional => 1
1033     },
1034     trans => {
1035         type => 'string',
1036         enum => [qw(none lba auto)],
1037         description => "Force disk geometry bios translation mode.",
1038         optional => 1,
1039     },
1040     snapshot => {
1041         type => 'boolean',
1042         description => "Controls qemu's snapshot mode feature."
1043             . " If activated, changes made to the disk are temporary and will"
1044             . " be discarded when the VM is shutdown.",
1045         optional => 1,
1046     },
1047     cache => {
1048         type => 'string',
1049         enum => [qw(none writethrough writeback unsafe directsync)],
1050         description => "The drive's cache mode",
1051         optional => 1,
1052     },
1053     format => get_standard_option('pve-qm-image-format'),
1054     size => {
1055         type => 'string',
1056         format => 'disk-size',
1057         format_description => 'DiskSize',
1058         description => "Disk size. This is purely informational and has no effect.",
1059         optional => 1,
1060     },
1061     backup => {
1062         type => 'boolean',
1063         description => "Whether the drive should be included when making backups.",
1064         optional => 1,
1065     },
1066     replicate => {
1067         type => 'boolean',
1068         description => 'Whether the drive should considered for replication jobs.',
1069         optional => 1,
1070         default => 1,
1071     },
1072     rerror => {
1073         type => 'string',
1074         enum => [qw(ignore report stop)],
1075         description => 'Read error action.',
1076         optional => 1,
1077     },
1078     werror => {
1079         type => 'string',
1080         enum => [qw(enospc ignore report stop)],
1081         description => 'Write error action.',
1082         optional => 1,
1083     },
1084     aio => {
1085         type => 'string',
1086         enum => [qw(native threads)],
1087         description => 'AIO type to use.',
1088         optional => 1,
1089     },
1090     discard => {
1091         type => 'string',
1092         enum => [qw(ignore on)],
1093         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1094         optional => 1,
1095     },
1096     detect_zeroes => {
1097         type => 'boolean',
1098         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1099         optional => 1,
1100     },
1101     serial => {
1102         type => 'string',
1103         format => 'urlencoded',
1104         format_description => 'serial',
1105         maxLength => 20*3, # *3 since it's %xx url enoded
1106         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1107         optional => 1,
1108     },
1109     shared => {
1110         type => 'boolean',
1111         description => 'Mark this locally-managed volume as available on all nodes',
1112         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1113         optional => 1,
1114         default => 0,
1115     }
1116 );
1117
1118 my %iothread_fmt = ( iothread => {
1119         type => 'boolean',
1120         description => "Whether to use iothreads for this drive",
1121         optional => 1,
1122 });
1123
1124 my %model_fmt = (
1125     model => {
1126         type => 'string',
1127         format => 'urlencoded',
1128         format_description => 'model',
1129         maxLength => 40*3, # *3 since it's %xx url enoded
1130         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1131         optional => 1,
1132     },
1133 );
1134
1135 my %queues_fmt = (
1136     queues => {
1137         type => 'integer',
1138         description => "Number of queues.",
1139         minimum => 2,
1140         optional => 1
1141     }
1142 );
1143
1144 my %scsiblock_fmt = (
1145     scsiblock => {
1146         type => 'boolean',
1147         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1148         optional => 1,
1149         default => 0,
1150     },
1151 );
1152
1153 my %ssd_fmt = (
1154     ssd => {
1155         type => 'boolean',
1156         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1157         optional => 1,
1158     },
1159 );
1160
1161 my %wwn_fmt = (
1162     wwn => {
1163         type => 'string',
1164         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1165         format_description => 'wwn',
1166         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1167         optional => 1,
1168     },
1169 );
1170
1171 my $add_throttle_desc = sub {
1172     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1173     my $d = {
1174         type => $type,
1175         format_description => $unit,
1176         description => "Maximum $what in $longunit.",
1177         optional => 1,
1178     };
1179     $d->{minimum} = $minimum if defined($minimum);
1180     $drivedesc_base{$key} = $d;
1181 };
1182 # throughput: (leaky bucket)
1183 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1184 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1186 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1187 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1189 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1190 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1191 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1192
1193 # pools: (pool of IO before throttling starts taking effect)
1194 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1195 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1197 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1198 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1199 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1200
1201 # burst lengths
1202 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1203 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1205 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1208
1209 # legacy support
1210 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1211 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1212 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1213 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1214
1215 my $ide_fmt = {
1216     %drivedesc_base,
1217     %model_fmt,
1218     %ssd_fmt,
1219     %wwn_fmt,
1220 };
1221 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1222
1223 my $idedesc = {
1224     optional => 1,
1225     type => 'string', format => $ide_fmt,
1226     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1227 };
1228 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1229
1230 my $scsi_fmt = {
1231     %drivedesc_base,
1232     %iothread_fmt,
1233     %queues_fmt,
1234     %scsiblock_fmt,
1235     %ssd_fmt,
1236     %wwn_fmt,
1237 };
1238 my $scsidesc = {
1239     optional => 1,
1240     type => 'string', format => $scsi_fmt,
1241     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1242 };
1243 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1244
1245 my $sata_fmt = {
1246     %drivedesc_base,
1247     %ssd_fmt,
1248     %wwn_fmt,
1249 };
1250 my $satadesc = {
1251     optional => 1,
1252     type => 'string', format => $sata_fmt,
1253     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1254 };
1255 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1256
1257 my $virtio_fmt = {
1258     %drivedesc_base,
1259     %iothread_fmt,
1260 };
1261 my $virtiodesc = {
1262     optional => 1,
1263     type => 'string', format => $virtio_fmt,
1264     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1265 };
1266 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1267
1268 my $alldrive_fmt = {
1269     %drivedesc_base,
1270     %iothread_fmt,
1271     %model_fmt,
1272     %queues_fmt,
1273     %scsiblock_fmt,
1274     %ssd_fmt,
1275     %wwn_fmt,
1276 };
1277
1278 my $efidisk_fmt = {
1279     volume => { alias => 'file' },
1280     file => {
1281         type => 'string',
1282         format => 'pve-volume-id-or-qm-path',
1283         default_key => 1,
1284         format_description => 'volume',
1285         description => "The drive's backing volume.",
1286     },
1287     format => get_standard_option('pve-qm-image-format'),
1288     size => {
1289         type => 'string',
1290         format => 'disk-size',
1291         format_description => 'DiskSize',
1292         description => "Disk size. This is purely informational and has no effect.",
1293         optional => 1,
1294     },
1295 };
1296
1297 my $efidisk_desc = {
1298     optional => 1,
1299     type => 'string', format => $efidisk_fmt,
1300     description => "Configure a Disk for storing EFI vars",
1301 };
1302
1303 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1304
1305 my $usb_fmt = {
1306     host => {
1307         default_key => 1,
1308         type => 'string', format => 'pve-qm-usb-device',
1309         format_description => 'HOSTUSBDEVICE|spice',
1310         description => <<EODESCR,
1311 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1312
1313  'bus-port(.port)*' (decimal numbers) or
1314  'vendor_id:product_id' (hexadeciaml numbers) or
1315  'spice'
1316
1317 You can use the 'lsusb -t' command to list existing usb devices.
1318
1319 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1320
1321 The value 'spice' can be used to add a usb redirection devices for spice.
1322 EODESCR
1323     },
1324     usb3 => {
1325         optional => 1,
1326         type => 'boolean',
1327         description => "Specifies whether if given host option is a USB3 device or port.",
1328         default => 0,
1329     },
1330 };
1331
1332 my $usbdesc = {
1333     optional => 1,
1334     type => 'string', format => $usb_fmt,
1335     description => "Configure an USB device (n is 0 to 4).",
1336 };
1337 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1338
1339 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1340 my $hostpci_fmt = {
1341     host => {
1342         default_key => 1,
1343         type => 'string',
1344         pattern => qr/$PCIRE(;$PCIRE)*/,
1345         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1346         description => <<EODESCR,
1347 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1348 of PCI virtual functions of the host. HOSTPCIID syntax is:
1349
1350 'bus:dev.func' (hexadecimal numbers)
1351
1352 You can us the 'lspci' command to list existing PCI devices.
1353 EODESCR
1354     },
1355     rombar => {
1356         type => 'boolean',
1357         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1358         optional => 1,
1359         default => 1,
1360     },
1361     romfile => {
1362         type => 'string',
1363         pattern => '[^,;]+',
1364         format_description => 'string',
1365         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1366         optional => 1,
1367     },
1368     pcie => {
1369         type => 'boolean',
1370         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1371         optional => 1,
1372         default => 0,
1373     },
1374     'x-vga' => {
1375         type => 'boolean',
1376         description =>  "Enable vfio-vga device support.",
1377         optional => 1,
1378         default => 0,
1379     },
1380     'mdev' => {
1381         type => 'string',
1382         format_description => 'string',
1383         pattern => '[^/\.:]+',
1384         optional => 1,
1385         description => <<EODESCR
1386 The type of mediated device to use.
1387 An instance of this type will be created on startup of the VM and
1388 will be cleaned up when the VM stops.
1389 EODESCR
1390     }
1391 };
1392 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1393
1394 my $hostpcidesc = {
1395         optional => 1,
1396         type => 'string', format => 'pve-qm-hostpci',
1397         description => "Map host PCI devices into guest.",
1398         verbose_description =>  <<EODESCR,
1399 Map host PCI devices into guest.
1400
1401 NOTE: This option allows direct access to host hardware. So it is no longer
1402 possible to migrate such machines - use with special care.
1403
1404 CAUTION: Experimental! User reported problems with this option.
1405 EODESCR
1406 };
1407 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1408
1409 my $serialdesc = {
1410         optional => 1,
1411         type => 'string',
1412         pattern => '(/dev/.+|socket)',
1413         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1414         verbose_description =>  <<EODESCR,
1415 Create a serial device inside the VM (n is 0 to 3), and pass through a
1416 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1417 host side (use 'qm terminal' to open a terminal connection).
1418
1419 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1420
1421 CAUTION: Experimental! User reported problems with this option.
1422 EODESCR
1423 };
1424
1425 my $paralleldesc= {
1426         optional => 1,
1427         type => 'string',
1428         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1429         description =>  "Map host parallel devices (n is 0 to 2).",
1430         verbose_description =>  <<EODESCR,
1431 Map host parallel devices (n is 0 to 2).
1432
1433 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1434
1435 CAUTION: Experimental! User reported problems with this option.
1436 EODESCR
1437 };
1438
1439 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1440     $confdesc->{"parallel$i"} = $paralleldesc;
1441 }
1442
1443 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1444     $confdesc->{"serial$i"} = $serialdesc;
1445 }
1446
1447 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1448     $confdesc->{"hostpci$i"} = $hostpcidesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1452     $drivename_hash->{"ide$i"} = 1;
1453     $confdesc->{"ide$i"} = $idedesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1457     $drivename_hash->{"sata$i"} = 1;
1458     $confdesc->{"sata$i"} = $satadesc;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1462     $drivename_hash->{"scsi$i"} = 1;
1463     $confdesc->{"scsi$i"} = $scsidesc ;
1464 }
1465
1466 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1467     $drivename_hash->{"virtio$i"} = 1;
1468     $confdesc->{"virtio$i"} = $virtiodesc;
1469 }
1470
1471 $drivename_hash->{efidisk0} = 1;
1472 $confdesc->{efidisk0} = $efidisk_desc;
1473
1474 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1475     $confdesc->{"usb$i"} = $usbdesc;
1476 }
1477
1478 my $unuseddesc = {
1479     optional => 1,
1480     type => 'string', format => 'pve-volume-id',
1481     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1482 };
1483
1484 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1485     $confdesc->{"unused$i"} = $unuseddesc;
1486 }
1487
1488 my $kvm_api_version = 0;
1489
1490 sub kvm_version {
1491     return $kvm_api_version if $kvm_api_version;
1492
1493     open my $fh, '<', '/dev/kvm'
1494         or return undef;
1495
1496     # 0xae00 => KVM_GET_API_VERSION
1497     $kvm_api_version = ioctl($fh, 0xae00, 0);
1498
1499     return $kvm_api_version;
1500 }
1501
1502 my $kvm_user_version = {};
1503 my $kvm_mtime = {};
1504
1505 sub kvm_user_version {
1506     my ($binary) = @_;
1507
1508     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1509     my $st = stat($binary);
1510
1511     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1512     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1513         $cachedmtime == $st->mtime;
1514
1515     $kvm_user_version->{$binary} = 'unknown';
1516     $kvm_mtime->{$binary} = $st->mtime;
1517
1518     my $code = sub {
1519         my $line = shift;
1520         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1521             $kvm_user_version->{$binary} = $2;
1522         }
1523     };
1524
1525     eval { run_command([$binary, '--version'], outfunc => $code); };
1526     warn $@ if $@;
1527
1528     return $kvm_user_version->{$binary};
1529
1530 }
1531
1532 sub kernel_has_vhost_net {
1533     return -c '/dev/vhost-net';
1534 }
1535
1536 sub valid_drive_names {
1537     # order is important - used to autoselect boot disk
1538     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1539             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1540             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1541             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1542             'efidisk0');
1543 }
1544
1545 sub is_valid_drivename {
1546     my $dev = shift;
1547
1548     return defined($drivename_hash->{$dev});
1549 }
1550
1551 sub option_exists {
1552     my $key = shift;
1553     return defined($confdesc->{$key});
1554 }
1555
1556 sub nic_models {
1557     return $nic_model_list;
1558 }
1559
1560 sub os_list_description {
1561
1562     return {
1563         other => 'Other',
1564         wxp => 'Windows XP',
1565         w2k => 'Windows 2000',
1566         w2k3 =>, 'Windows 2003',
1567         w2k8 => 'Windows 2008',
1568         wvista => 'Windows Vista',
1569         win7 => 'Windows 7',
1570         win8 => 'Windows 8/2012',
1571         win10 => 'Windows 10/2016',
1572         l24 => 'Linux 2.4',
1573         l26 => 'Linux 2.6',
1574     };
1575 }
1576
1577 my $cdrom_path;
1578
1579 sub get_cdrom_path {
1580
1581     return  $cdrom_path if $cdrom_path;
1582
1583     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1584     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1585     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1586 }
1587
1588 sub get_iso_path {
1589     my ($storecfg, $vmid, $cdrom) = @_;
1590
1591     if ($cdrom eq 'cdrom') {
1592         return get_cdrom_path();
1593     } elsif ($cdrom eq 'none') {
1594         return '';
1595     } elsif ($cdrom =~ m|^/|) {
1596         return $cdrom;
1597     } else {
1598         return PVE::Storage::path($storecfg, $cdrom);
1599     }
1600 }
1601
1602 # try to convert old style file names to volume IDs
1603 sub filename_to_volume_id {
1604     my ($vmid, $file, $media) = @_;
1605
1606      if (!($file eq 'none' || $file eq 'cdrom' ||
1607           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1608
1609         return undef if $file =~ m|/|;
1610
1611         if ($media && $media eq 'cdrom') {
1612             $file = "local:iso/$file";
1613         } else {
1614             $file = "local:$vmid/$file";
1615         }
1616     }
1617
1618     return $file;
1619 }
1620
1621 sub verify_media_type {
1622     my ($opt, $vtype, $media) = @_;
1623
1624     return if !$media;
1625
1626     my $etype;
1627     if ($media eq 'disk') {
1628         $etype = 'images';
1629     } elsif ($media eq 'cdrom') {
1630         $etype = 'iso';
1631     } else {
1632         die "internal error";
1633     }
1634
1635     return if ($vtype eq $etype);
1636
1637     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1638 }
1639
1640 sub cleanup_drive_path {
1641     my ($opt, $storecfg, $drive) = @_;
1642
1643     # try to convert filesystem paths to volume IDs
1644
1645     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1646         ($drive->{file} !~ m|^/dev/.+|) &&
1647         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1648         ($drive->{file} !~ m/^\d+$/)) {
1649         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1650         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1651         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1652         verify_media_type($opt, $vtype, $drive->{media});
1653         $drive->{file} = $volid;
1654     }
1655
1656     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1657 }
1658
1659 sub parse_hotplug_features {
1660     my ($data) = @_;
1661
1662     my $res = {};
1663
1664     return $res if $data eq '0';
1665
1666     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1667
1668     foreach my $feature (PVE::Tools::split_list($data)) {
1669         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1670             $res->{$1} = 1;
1671         } else {
1672             die "invalid hotplug feature '$feature'\n";
1673         }
1674     }
1675     return $res;
1676 }
1677
1678 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1679 sub pve_verify_hotplug_features {
1680     my ($value, $noerr) = @_;
1681
1682     return $value if parse_hotplug_features($value);
1683
1684     return undef if $noerr;
1685
1686     die "unable to parse hotplug option\n";
1687 }
1688
1689 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1690 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1691 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1692 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1693 #        [,iothread=on][,serial=serial][,model=model]
1694
1695 sub parse_drive {
1696     my ($key, $data) = @_;
1697
1698     my ($interface, $index);
1699
1700     if ($key =~ m/^([^\d]+)(\d+)$/) {
1701         $interface = $1;
1702         $index = $2;
1703     } else {
1704         return undef;
1705     }
1706
1707     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1708                                      : $confdesc->{$key}->{format};
1709     if (!$desc) {
1710         warn "invalid drive key: $key\n";
1711         return undef;
1712     }
1713     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1714     return undef if !$res;
1715     $res->{interface} = $interface;
1716     $res->{index} = $index;
1717
1718     my $error = 0;
1719     foreach my $opt (qw(bps bps_rd bps_wr)) {
1720         if (my $bps = defined(delete $res->{$opt})) {
1721             if (defined($res->{"m$opt"})) {
1722                 warn "both $opt and m$opt specified\n";
1723                 ++$error;
1724                 next;
1725             }
1726             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1727         }
1728     }
1729
1730     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1731     for my $requirement (
1732         [mbps_max => 'mbps'],
1733         [mbps_rd_max => 'mbps_rd'],
1734         [mbps_wr_max => 'mbps_wr'],
1735         [miops_max => 'miops'],
1736         [miops_rd_max => 'miops_rd'],
1737         [miops_wr_max => 'miops_wr'],
1738         [bps_max_length => 'mbps_max'],
1739         [bps_rd_max_length => 'mbps_rd_max'],
1740         [bps_wr_max_length => 'mbps_wr_max'],
1741         [iops_max_length => 'iops_max'],
1742         [iops_rd_max_length => 'iops_rd_max'],
1743         [iops_wr_max_length => 'iops_wr_max']) {
1744         my ($option, $requires) = @$requirement;
1745         if ($res->{$option} && !$res->{$requires}) {
1746             warn "$option requires $requires\n";
1747             ++$error;
1748         }
1749     }
1750
1751     return undef if $error;
1752
1753     return undef if $res->{mbps_rd} && $res->{mbps};
1754     return undef if $res->{mbps_wr} && $res->{mbps};
1755     return undef if $res->{iops_rd} && $res->{iops};
1756     return undef if $res->{iops_wr} && $res->{iops};
1757
1758     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1759         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1760         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1761         return undef if $res->{interface} eq 'virtio';
1762     }
1763
1764     if (my $size = $res->{size}) {
1765         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1766     }
1767
1768     return $res;
1769 }
1770
1771 sub print_drive {
1772     my ($vmid, $drive) = @_;
1773     my $data = { %$drive };
1774     delete $data->{$_} for qw(index interface);
1775     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1776 }
1777
1778 sub scsi_inquiry {
1779     my($fh, $noerr) = @_;
1780
1781     my $SG_IO = 0x2285;
1782     my $SG_GET_VERSION_NUM = 0x2282;
1783
1784     my $versionbuf = "\x00" x 8;
1785     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1786     if (!$ret) {
1787         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1788         return undef;
1789     }
1790     my $version = unpack("I", $versionbuf);
1791     if ($version < 30000) {
1792         die "scsi generic interface too old\n"  if !$noerr;
1793         return undef;
1794     }
1795
1796     my $buf = "\x00" x 36;
1797     my $sensebuf = "\x00" x 8;
1798     my $cmd = pack("C x3 C x1", 0x12, 36);
1799
1800     # see /usr/include/scsi/sg.h
1801     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1802
1803     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1804                       length($sensebuf), 0, length($buf), $buf,
1805                       $cmd, $sensebuf, 6000);
1806
1807     $ret = ioctl($fh, $SG_IO, $packet);
1808     if (!$ret) {
1809         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1810         return undef;
1811     }
1812
1813     my @res = unpack($sg_io_hdr_t, $packet);
1814     if ($res[17] || $res[18]) {
1815         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1816         return undef;
1817     }
1818
1819     my $res = {};
1820     (my $byte0, my $byte1, $res->{vendor},
1821      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1822
1823     $res->{removable} = $byte1 & 128 ? 1 : 0;
1824     $res->{type} = $byte0 & 31;
1825
1826     return $res;
1827 }
1828
1829 sub path_is_scsi {
1830     my ($path) = @_;
1831
1832     my $fh = IO::File->new("+<$path") || return undef;
1833     my $res = scsi_inquiry($fh, 1);
1834     close($fh);
1835
1836     return $res;
1837 }
1838
1839 sub machine_type_is_q35 {
1840     my ($conf) = @_;
1841
1842     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1843 }
1844
1845 sub print_tabletdevice_full {
1846     my ($conf, $arch) = @_;
1847
1848     my $q35 = machine_type_is_q35($conf);
1849
1850     # we use uhci for old VMs because tablet driver was buggy in older qemu
1851     my $usbbus;
1852     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1853         $usbbus = 'ehci';
1854     } else {
1855         $usbbus = 'uhci';
1856     }
1857
1858     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1859 }
1860
1861 sub print_keyboarddevice_full {
1862     my ($conf, $arch, $machine) = @_;
1863
1864     return undef if $arch ne 'aarch64';
1865
1866     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1867 }
1868
1869 sub print_drivedevice_full {
1870     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1871
1872     my $device = '';
1873     my $maxdev = 0;
1874
1875     if ($drive->{interface} eq 'virtio') {
1876         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1877         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1878         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1879     } elsif ($drive->{interface} eq 'scsi') {
1880
1881         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1882         my $unit = $drive->{index} % $maxdev;
1883         my $devicetype = 'hd';
1884         my $path = '';
1885         if (drive_is_cdrom($drive)) {
1886             $devicetype = 'cd';
1887         } else {
1888             if ($drive->{file} =~ m|^/|) {
1889                 $path = $drive->{file};
1890                 if (my $info = path_is_scsi($path)) {
1891                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1892                         $devicetype = 'block';
1893                     } elsif ($info->{type} == 1) { # tape
1894                         $devicetype = 'generic';
1895                     }
1896                 }
1897             } else {
1898                  $path = PVE::Storage::path($storecfg, $drive->{file});
1899             }
1900
1901             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1902             if ($path =~ m/^iscsi\:\/\// &&
1903                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1904                 $devicetype = 'generic';
1905             }
1906         }
1907
1908         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1909             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1910         } else {
1911             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1912         }
1913
1914         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1915             $device .= ",rotation_rate=1";
1916         }
1917         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1918
1919     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1920         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1921         my $controller = int($drive->{index} / $maxdev);
1922         my $unit = $drive->{index} % $maxdev;
1923         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1924
1925         $device = "ide-$devicetype";
1926         if ($drive->{interface} eq 'ide') {
1927             $device .= ",bus=ide.$controller,unit=$unit";
1928         } else {
1929             $device .= ",bus=ahci$controller.$unit";
1930         }
1931         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1932
1933         if ($devicetype eq 'hd') {
1934             if (my $model = $drive->{model}) {
1935                 $model = URI::Escape::uri_unescape($model);
1936                 $device .= ",model=$model";
1937             }
1938             if ($drive->{ssd}) {
1939                 $device .= ",rotation_rate=1";
1940             }
1941         }
1942         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1943     } elsif ($drive->{interface} eq 'usb') {
1944         die "implement me";
1945         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1946     } else {
1947         die "unsupported interface type";
1948     }
1949
1950     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1951
1952     if (my $serial = $drive->{serial}) {
1953         $serial = URI::Escape::uri_unescape($serial);
1954         $device .= ",serial=$serial";
1955     }
1956
1957
1958     return $device;
1959 }
1960
1961 sub get_initiator_name {
1962     my $initiator;
1963
1964     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1965     while (defined(my $line = <$fh>)) {
1966         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1967         $initiator = $1;
1968         last;
1969     }
1970     $fh->close();
1971
1972     return $initiator;
1973 }
1974
1975 sub print_drive_full {
1976     my ($storecfg, $vmid, $drive) = @_;
1977
1978     my $path;
1979     my $volid = $drive->{file};
1980     my $format;
1981
1982     if (drive_is_cdrom($drive)) {
1983         $path = get_iso_path($storecfg, $vmid, $volid);
1984     } else {
1985         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1986         if ($storeid) {
1987             $path = PVE::Storage::path($storecfg, $volid);
1988             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1989             $format = qemu_img_format($scfg, $volname);
1990         } else {
1991             $path = $volid;
1992             $format = "raw";
1993         }
1994    }
1995
1996     my $opts = '';
1997     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1998     foreach my $o (@qemu_drive_options) {
1999         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2000     }
2001
2002     # snapshot only accepts on|off
2003     if (defined($drive->{snapshot})) {
2004         my $v = $drive->{snapshot} ? 'on' : 'off';
2005         $opts .= ",snapshot=$v";
2006     }
2007
2008     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2009         my ($dir, $qmpname) = @$type;
2010         if (my $v = $drive->{"mbps$dir"}) {
2011             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2012         }
2013         if (my $v = $drive->{"mbps${dir}_max"}) {
2014             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2015         }
2016         if (my $v = $drive->{"bps${dir}_max_length"}) {
2017             $opts .= ",throttling.bps$qmpname-max-length=$v";
2018         }
2019         if (my $v = $drive->{"iops${dir}"}) {
2020             $opts .= ",throttling.iops$qmpname=$v";
2021         }
2022         if (my $v = $drive->{"iops${dir}_max"}) {
2023             $opts .= ",throttling.iops$qmpname-max=$v";
2024         }
2025         if (my $v = $drive->{"iops${dir}_max_length"}) {
2026             $opts .= ",throttling.iops$qmpname-max-length=$v";
2027         }
2028     }
2029
2030     $opts .= ",format=$format" if $format && !$drive->{format};
2031
2032     my $cache_direct = 0;
2033
2034     if (my $cache = $drive->{cache}) {
2035         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2036     } elsif (!drive_is_cdrom($drive)) {
2037         $opts .= ",cache=none";
2038         $cache_direct = 1;
2039     }
2040
2041     # aio native works only with O_DIRECT
2042     if (!$drive->{aio}) {
2043         if($cache_direct) {
2044             $opts .= ",aio=native";
2045         } else {
2046             $opts .= ",aio=threads";
2047         }
2048     }
2049
2050     if (!drive_is_cdrom($drive)) {
2051         my $detectzeroes;
2052         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2053             $detectzeroes = 'off';
2054         } elsif ($drive->{discard}) {
2055             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2056         } else {
2057             # This used to be our default with discard not being specified:
2058             $detectzeroes = 'on';
2059         }
2060         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2061     }
2062
2063     my $pathinfo = $path ? "file=$path," : '';
2064
2065     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2066 }
2067
2068 sub print_netdevice_full {
2069     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2070
2071     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2072
2073     my $device = $net->{model};
2074     if ($net->{model} eq 'virtio') {
2075          $device = 'virtio-net-pci';
2076      };
2077
2078     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2079     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2080     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2081         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2082         my $vectors = $net->{queues} * 2 + 2;
2083         $tmpstr .= ",vectors=$vectors,mq=on";
2084     }
2085     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2086
2087     if ($use_old_bios_files) {
2088         my $romfile;
2089         if ($device eq 'virtio-net-pci') {
2090             $romfile = 'pxe-virtio.rom';
2091         } elsif ($device eq 'e1000') {
2092             $romfile = 'pxe-e1000.rom';
2093         } elsif ($device eq 'ne2k') {
2094             $romfile = 'pxe-ne2k_pci.rom';
2095         } elsif ($device eq 'pcnet') {
2096             $romfile = 'pxe-pcnet.rom';
2097         } elsif ($device eq 'rtl8139') {
2098             $romfile = 'pxe-rtl8139.rom';
2099         }
2100         $tmpstr .= ",romfile=$romfile" if $romfile;
2101     }
2102
2103     return $tmpstr;
2104 }
2105
2106 sub print_netdev_full {
2107     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2108
2109     my $i = '';
2110     if ($netid =~ m/^net(\d+)$/) {
2111         $i = int($1);
2112     }
2113
2114     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2115
2116     my $ifname = "tap${vmid}i$i";
2117
2118     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2119     die "interface name '$ifname' is too long (max 15 character)\n"
2120         if length($ifname) >= 16;
2121
2122     my $vhostparam = '';
2123     if (is_native($arch)) {
2124         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2125     }
2126
2127     my $vmname = $conf->{name} || "vm$vmid";
2128
2129     my $netdev = "";
2130     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2131
2132     if ($net->{bridge}) {
2133         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2134     } else {
2135         $netdev = "type=user,id=$netid,hostname=$vmname";
2136     }
2137
2138     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2139
2140     return $netdev;
2141 }
2142
2143
2144 sub print_cpu_device {
2145     my ($conf, $id) = @_;
2146
2147     my $kvm = $conf->{kvm} // 1;
2148     my $cpu = $kvm ? "kvm64" : "qemu64";
2149     if (my $cputype = $conf->{cpu}) {
2150         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2151             or die "Cannot parse cpu description: $cputype\n";
2152         $cpu = $cpuconf->{cputype};
2153     }
2154
2155     my $cores = $conf->{cores} || 1;
2156
2157     my $current_core = ($id - 1) % $cores;
2158     my $current_socket = int(($id - 1 - $current_core)/$cores);
2159
2160     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2161 }
2162
2163 my $vga_map = {
2164     'cirrus' => 'cirrus-vga',
2165     'std' => 'VGA',
2166     'vmware' => 'vmware-svga',
2167     'virtio' => 'virtio-vga',
2168 };
2169
2170 sub print_vga_device {
2171     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2172
2173     my $type = $vga_map->{$vga->{type}};
2174     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2175         $type = 'virtio-gpu';
2176     }
2177     my $vgamem_mb = $vga->{memory};
2178     if ($qxlnum) {
2179         $type = $id ? 'qxl' : 'qxl-vga';
2180     }
2181     die "no devicetype for $vga->{type}\n" if !$type;
2182
2183     my $memory = "";
2184     if ($vgamem_mb) {
2185         if ($vga->{type} eq 'virtio') {
2186             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2187             $memory = ",max_hostmem=$bytes";
2188         } elsif ($qxlnum) {
2189             # from https://www.spice-space.org/multiple-monitors.html
2190             $memory = ",vgamem_mb=$vga->{memory}";
2191             my $ram = $vgamem_mb * 4;
2192             my $vram = $vgamem_mb * 2;
2193             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2194         } else {
2195             $memory = ",vgamem_mb=$vga->{memory}";
2196         }
2197     } elsif ($qxlnum && $id) {
2198         $memory = ",ram_size=67108864,vram_size=33554432";
2199     }
2200
2201     my $q35 = machine_type_is_q35($conf);
2202     my $vgaid = "vga" . ($id // '');
2203     my $pciaddr;
2204
2205     if ($q35 && $vgaid eq 'vga') {
2206         # the first display uses pcie.0 bus on q35 machines
2207         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2208     } else {
2209         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2210     }
2211
2212     return "$type,id=${vgaid}${memory}${pciaddr}";
2213 }
2214
2215 sub drive_is_cloudinit {
2216     my ($drive) = @_;
2217     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2218 }
2219
2220 sub drive_is_cdrom {
2221     my ($drive, $exclude_cloudinit) = @_;
2222
2223     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2224
2225     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2226
2227 }
2228
2229 sub parse_number_sets {
2230     my ($set) = @_;
2231     my $res = [];
2232     foreach my $part (split(/;/, $set)) {
2233         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2234             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2235             push @$res, [ $1, $2 ];
2236         } else {
2237             die "invalid range: $part\n";
2238         }
2239     }
2240     return $res;
2241 }
2242
2243 sub parse_numa {
2244     my ($data) = @_;
2245
2246     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2247     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2248     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2249     return $res;
2250 }
2251
2252 sub parse_hostpci {
2253     my ($value) = @_;
2254
2255     return undef if !$value;
2256
2257     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2258
2259     my @idlist = split(/;/, $res->{host});
2260     delete $res->{host};
2261     foreach my $id (@idlist) {
2262         if ($id =~ m/\./) { # full id 00:00.1
2263             push @{$res->{pciid}}, {
2264                 id => $id,
2265             };
2266         } else { # partial id 00:00
2267             $res->{pciid} = PVE::SysFSTools::lspci($id);
2268         }
2269     }
2270     return $res;
2271 }
2272
2273 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2274 sub parse_net {
2275     my ($data) = @_;
2276
2277     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2278     if ($@) {
2279         warn $@;
2280         return undef;
2281     }
2282     if (!defined($res->{macaddr})) {
2283         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2284         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2285     }
2286     return $res;
2287 }
2288
2289 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2290 sub parse_ipconfig {
2291     my ($data) = @_;
2292
2293     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2294     if ($@) {
2295         warn $@;
2296         return undef;
2297     }
2298
2299     if ($res->{gw} && !$res->{ip}) {
2300         warn 'gateway specified without specifying an IP address';
2301         return undef;
2302     }
2303     if ($res->{gw6} && !$res->{ip6}) {
2304         warn 'IPv6 gateway specified without specifying an IPv6 address';
2305         return undef;
2306     }
2307     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2308         warn 'gateway specified together with DHCP';
2309         return undef;
2310     }
2311     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2312         # gw6 + auto/dhcp
2313         warn "IPv6 gateway specified together with $res->{ip6} address";
2314         return undef;
2315     }
2316
2317     if (!$res->{ip} && !$res->{ip6}) {
2318         return { ip => 'dhcp', ip6 => 'dhcp' };
2319     }
2320
2321     return $res;
2322 }
2323
2324 sub print_net {
2325     my $net = shift;
2326
2327     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2328 }
2329
2330 sub add_random_macs {
2331     my ($settings) = @_;
2332
2333     foreach my $opt (keys %$settings) {
2334         next if $opt !~ m/^net(\d+)$/;
2335         my $net = parse_net($settings->{$opt});
2336         next if !$net;
2337         $settings->{$opt} = print_net($net);
2338     }
2339 }
2340
2341 sub vm_is_volid_owner {
2342     my ($storecfg, $vmid, $volid) = @_;
2343
2344     if ($volid !~  m|^/|) {
2345         my ($path, $owner);
2346         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2347         if ($owner && ($owner == $vmid)) {
2348             return 1;
2349         }
2350     }
2351
2352     return undef;
2353 }
2354
2355 sub vmconfig_register_unused_drive {
2356     my ($storecfg, $vmid, $conf, $drive) = @_;
2357
2358     if (drive_is_cloudinit($drive)) {
2359         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2360         warn $@ if $@;
2361     } elsif (!drive_is_cdrom($drive)) {
2362         my $volid = $drive->{file};
2363         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2364             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2365         }
2366     }
2367 }
2368
2369 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2370 my $smbios1_fmt = {
2371     uuid => {
2372         type => 'string',
2373         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2374         format_description => 'UUID',
2375         description => "Set SMBIOS1 UUID.",
2376         optional => 1,
2377     },
2378     version => {
2379         type => 'string',
2380         pattern => '[A-Za-z0-9+\/]+={0,2}',
2381         format_description => 'Base64 encoded string',
2382         description => "Set SMBIOS1 version.",
2383         optional => 1,
2384     },
2385     serial => {
2386         type => 'string',
2387         pattern => '[A-Za-z0-9+\/]+={0,2}',
2388         format_description => 'Base64 encoded string',
2389         description => "Set SMBIOS1 serial number.",
2390         optional => 1,
2391     },
2392     manufacturer => {
2393         type => 'string',
2394         pattern => '[A-Za-z0-9+\/]+={0,2}',
2395         format_description => 'Base64 encoded string',
2396         description => "Set SMBIOS1 manufacturer.",
2397         optional => 1,
2398     },
2399     product => {
2400         type => 'string',
2401         pattern => '[A-Za-z0-9+\/]+={0,2}',
2402         format_description => 'Base64 encoded string',
2403         description => "Set SMBIOS1 product ID.",
2404         optional => 1,
2405     },
2406     sku => {
2407         type => 'string',
2408         pattern => '[A-Za-z0-9+\/]+={0,2}',
2409         format_description => 'Base64 encoded string',
2410         description => "Set SMBIOS1 SKU string.",
2411         optional => 1,
2412     },
2413     family => {
2414         type => 'string',
2415         pattern => '[A-Za-z0-9+\/]+={0,2}',
2416         format_description => 'Base64 encoded string',
2417         description => "Set SMBIOS1 family string.",
2418         optional => 1,
2419     },
2420     base64 => {
2421         type => 'boolean',
2422         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2423         optional => 1,
2424     },
2425 };
2426
2427 sub parse_smbios1 {
2428     my ($data) = @_;
2429
2430     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2431     warn $@ if $@;
2432     return $res;
2433 }
2434
2435 sub print_smbios1 {
2436     my ($smbios1) = @_;
2437     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2438 }
2439
2440 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2441
2442 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2443 sub verify_bootdisk {
2444     my ($value, $noerr) = @_;
2445
2446     return $value if is_valid_drivename($value);
2447
2448     return undef if $noerr;
2449
2450     die "invalid boot disk '$value'\n";
2451 }
2452
2453 sub parse_watchdog {
2454     my ($value) = @_;
2455
2456     return undef if !$value;
2457
2458     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2459     warn $@ if $@;
2460     return $res;
2461 }
2462
2463 sub parse_guest_agent {
2464     my ($value) = @_;
2465
2466     return {} if !defined($value->{agent});
2467
2468     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2469     warn $@ if $@;
2470
2471     # if the agent is disabled ignore the other potentially set properties
2472     return {} if !$res->{enabled};
2473     return $res;
2474 }
2475
2476 sub parse_vga {
2477     my ($value) = @_;
2478
2479     return {} if !$value;
2480     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2481     warn $@ if $@;
2482     return $res;
2483 }
2484
2485 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2486 sub verify_usb_device {
2487     my ($value, $noerr) = @_;
2488
2489     return $value if parse_usb_device($value);
2490
2491     return undef if $noerr;
2492
2493     die "unable to parse usb device\n";
2494 }
2495
2496 # add JSON properties for create and set function
2497 sub json_config_properties {
2498     my $prop = shift;
2499
2500     foreach my $opt (keys %$confdesc) {
2501         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2502         $prop->{$opt} = $confdesc->{$opt};
2503     }
2504
2505     return $prop;
2506 }
2507
2508 # return copy of $confdesc_cloudinit to generate documentation
2509 sub cloudinit_config_properties {
2510
2511     return dclone($confdesc_cloudinit);
2512 }
2513
2514 sub check_type {
2515     my ($key, $value) = @_;
2516
2517     die "unknown setting '$key'\n" if !$confdesc->{$key};
2518
2519     my $type = $confdesc->{$key}->{type};
2520
2521     if (!defined($value)) {
2522         die "got undefined value\n";
2523     }
2524
2525     if ($value =~ m/[\n\r]/) {
2526         die "property contains a line feed\n";
2527     }
2528
2529     if ($type eq 'boolean') {
2530         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2531         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2532         die "type check ('boolean') failed - got '$value'\n";
2533     } elsif ($type eq 'integer') {
2534         return int($1) if $value =~ m/^(\d+)$/;
2535         die "type check ('integer') failed - got '$value'\n";
2536     } elsif ($type eq 'number') {
2537         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2538         die "type check ('number') failed - got '$value'\n";
2539     } elsif ($type eq 'string') {
2540         if (my $fmt = $confdesc->{$key}->{format}) {
2541             PVE::JSONSchema::check_format($fmt, $value);
2542             return $value;
2543         }
2544         $value =~ s/^\"(.*)\"$/$1/;
2545         return $value;
2546     } else {
2547         die "internal error"
2548     }
2549 }
2550
2551 sub destroy_vm {
2552     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2553
2554     my $conf = PVE::QemuConfig->load_config($vmid);
2555
2556     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2557
2558     if ($conf->{template}) {
2559         # check if any base image is still used by a linked clone
2560         foreach_drive($conf, sub {
2561                 my ($ds, $drive) = @_;
2562
2563                 return if drive_is_cdrom($drive);
2564
2565                 my $volid = $drive->{file};
2566
2567                 return if !$volid || $volid =~ m|^/|;
2568
2569                 die "base volume '$volid' is still in use by linked cloned\n"
2570                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2571
2572         });
2573     }
2574
2575     # only remove disks owned by this VM
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578
2579         return if drive_is_cdrom($drive, 1);
2580
2581         my $volid = $drive->{file};
2582
2583         return if !$volid || $volid =~ m|^/|;
2584
2585         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2586         return if !$path || !$owner || ($owner != $vmid);
2587
2588         eval {
2589             PVE::Storage::vdisk_free($storecfg, $volid);
2590         };
2591         warn "Could not remove disk '$volid', check manually: $@" if $@;
2592
2593     });
2594
2595     if ($keep_empty_config) {
2596         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2597     } else {
2598         PVE::QemuConfig->destroy_config($vmid);
2599     }
2600
2601     # also remove unused disk
2602     eval {
2603         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2604
2605         eval {
2606             PVE::Storage::foreach_volid($dl, sub {
2607                 my ($volid, $sid, $volname, $d) = @_;
2608                 PVE::Storage::vdisk_free($storecfg, $volid);
2609             });
2610         };
2611         warn $@ if $@;
2612
2613     };
2614     warn $@ if $@;
2615 }
2616
2617 sub parse_vm_config {
2618     my ($filename, $raw) = @_;
2619
2620     return undef if !defined($raw);
2621
2622     my $res = {
2623         digest => Digest::SHA::sha1_hex($raw),
2624         snapshots => {},
2625         pending => {},
2626     };
2627
2628     $filename =~ m|/qemu-server/(\d+)\.conf$|
2629         || die "got strange filename '$filename'";
2630
2631     my $vmid = $1;
2632
2633     my $conf = $res;
2634     my $descr;
2635     my $section = '';
2636
2637     my @lines = split(/\n/, $raw);
2638     foreach my $line (@lines) {
2639         next if $line =~ m/^\s*$/;
2640
2641         if ($line =~ m/^\[PENDING\]\s*$/i) {
2642             $section = 'pending';
2643             if (defined($descr)) {
2644                 $descr =~ s/\s+$//;
2645                 $conf->{description} = $descr;
2646             }
2647             $descr = undef;
2648             $conf = $res->{$section} = {};
2649             next;
2650
2651         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2652             $section = $1;
2653             if (defined($descr)) {
2654                 $descr =~ s/\s+$//;
2655                 $conf->{description} = $descr;
2656             }
2657             $descr = undef;
2658             $conf = $res->{snapshots}->{$section} = {};
2659             next;
2660         }
2661
2662         if ($line =~ m/^\#(.*)\s*$/) {
2663             $descr = '' if !defined($descr);
2664             $descr .= PVE::Tools::decode_text($1) . "\n";
2665             next;
2666         }
2667
2668         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2669             $descr = '' if !defined($descr);
2670             $descr .= PVE::Tools::decode_text($2);
2671         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2672             $conf->{snapstate} = $1;
2673         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2674             my $key = $1;
2675             my $value = $2;
2676             $conf->{$key} = $value;
2677         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2678             my $value = $1;
2679             if ($section eq 'pending') {
2680                 $conf->{delete} = $value; # we parse this later
2681             } else {
2682                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2683             }
2684         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2685             my $key = $1;
2686             my $value = $2;
2687             eval { $value = check_type($key, $value); };
2688             if ($@) {
2689                 warn "vm $vmid - unable to parse value of '$key' - $@";
2690             } else {
2691                 $key = 'ide2' if $key eq 'cdrom';
2692                 my $fmt = $confdesc->{$key}->{format};
2693                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2694                     my $v = parse_drive($key, $value);
2695                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2696                         $v->{file} = $volid;
2697                         $value = print_drive($vmid, $v);
2698                     } else {
2699                         warn "vm $vmid - unable to parse value of '$key'\n";
2700                         next;
2701                     }
2702                 }
2703
2704                 $conf->{$key} = $value;
2705             }
2706         }
2707     }
2708
2709     if (defined($descr)) {
2710         $descr =~ s/\s+$//;
2711         $conf->{description} = $descr;
2712     }
2713     delete $res->{snapstate}; # just to be sure
2714
2715     return $res;
2716 }
2717
2718 sub write_vm_config {
2719     my ($filename, $conf) = @_;
2720
2721     delete $conf->{snapstate}; # just to be sure
2722
2723     if ($conf->{cdrom}) {
2724         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2725         $conf->{ide2} = $conf->{cdrom};
2726         delete $conf->{cdrom};
2727     }
2728
2729     # we do not use 'smp' any longer
2730     if ($conf->{sockets}) {
2731         delete $conf->{smp};
2732     } elsif ($conf->{smp}) {
2733         $conf->{sockets} = $conf->{smp};
2734         delete $conf->{cores};
2735         delete $conf->{smp};
2736     }
2737
2738     my $used_volids = {};
2739
2740     my $cleanup_config = sub {
2741         my ($cref, $pending, $snapname) = @_;
2742
2743         foreach my $key (keys %$cref) {
2744             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2745                 $key eq 'snapstate' || $key eq 'pending';
2746             my $value = $cref->{$key};
2747             if ($key eq 'delete') {
2748                 die "propertry 'delete' is only allowed in [PENDING]\n"
2749                     if !$pending;
2750                 # fixme: check syntax?
2751                 next;
2752             }
2753             eval { $value = check_type($key, $value); };
2754             die "unable to parse value of '$key' - $@" if $@;
2755
2756             $cref->{$key} = $value;
2757
2758             if (!$snapname && is_valid_drivename($key)) {
2759                 my $drive = parse_drive($key, $value);
2760                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2761             }
2762         }
2763     };
2764
2765     &$cleanup_config($conf);
2766
2767     &$cleanup_config($conf->{pending}, 1);
2768
2769     foreach my $snapname (keys %{$conf->{snapshots}}) {
2770         die "internal error" if $snapname eq 'pending';
2771         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2772     }
2773
2774     # remove 'unusedX' settings if we re-add a volume
2775     foreach my $key (keys %$conf) {
2776         my $value = $conf->{$key};
2777         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2778             delete $conf->{$key};
2779         }
2780     }
2781
2782     my $generate_raw_config = sub {
2783         my ($conf, $pending) = @_;
2784
2785         my $raw = '';
2786
2787         # add description as comment to top of file
2788         if (defined(my $descr = $conf->{description})) {
2789             if ($descr) {
2790                 foreach my $cl (split(/\n/, $descr)) {
2791                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2792                 }
2793             } else {
2794                 $raw .= "#\n" if $pending;
2795             }
2796         }
2797
2798         foreach my $key (sort keys %$conf) {
2799             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2800             $raw .= "$key: $conf->{$key}\n";
2801         }
2802         return $raw;
2803     };
2804
2805     my $raw = &$generate_raw_config($conf);
2806
2807     if (scalar(keys %{$conf->{pending}})){
2808         $raw .= "\n[PENDING]\n";
2809         $raw .= &$generate_raw_config($conf->{pending}, 1);
2810     }
2811
2812     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2813         $raw .= "\n[$snapname]\n";
2814         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2815     }
2816
2817     return $raw;
2818 }
2819
2820 sub load_defaults {
2821
2822     my $res = {};
2823
2824     # we use static defaults from our JSON schema configuration
2825     foreach my $key (keys %$confdesc) {
2826         if (defined(my $default = $confdesc->{$key}->{default})) {
2827             $res->{$key} = $default;
2828         }
2829     }
2830
2831     return $res;
2832 }
2833
2834 sub config_list {
2835     my $vmlist = PVE::Cluster::get_vmlist();
2836     my $res = {};
2837     return $res if !$vmlist || !$vmlist->{ids};
2838     my $ids = $vmlist->{ids};
2839
2840     foreach my $vmid (keys %$ids) {
2841         my $d = $ids->{$vmid};
2842         next if !$d->{node} || $d->{node} ne $nodename;
2843         next if !$d->{type} || $d->{type} ne 'qemu';
2844         $res->{$vmid}->{exists} = 1;
2845     }
2846     return $res;
2847 }
2848
2849 # test if VM uses local resources (to prevent migration)
2850 sub check_local_resources {
2851     my ($conf, $noerr) = @_;
2852
2853     my @loc_res = ();
2854
2855     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2856     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2857
2858     push @loc_res, "ivshmem" if $conf->{ivshmem};
2859
2860     foreach my $k (keys %$conf) {
2861         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2862         # sockets are safe: they will recreated be on the target side post-migrate
2863         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2864         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2865     }
2866
2867     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2868
2869     return \@loc_res;
2870 }
2871
2872 # check if used storages are available on all nodes (use by migrate)
2873 sub check_storage_availability {
2874     my ($storecfg, $conf, $node) = @_;
2875
2876     foreach_drive($conf, sub {
2877         my ($ds, $drive) = @_;
2878
2879         my $volid = $drive->{file};
2880         return if !$volid;
2881
2882         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2883         return if !$sid;
2884
2885         # check if storage is available on both nodes
2886         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2887         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2888    });
2889 }
2890
2891 # list nodes where all VM images are available (used by has_feature API)
2892 sub shared_nodes {
2893     my ($conf, $storecfg) = @_;
2894
2895     my $nodelist = PVE::Cluster::get_nodelist();
2896     my $nodehash = { map { $_ => 1 } @$nodelist };
2897     my $nodename = PVE::INotify::nodename();
2898
2899     foreach_drive($conf, sub {
2900         my ($ds, $drive) = @_;
2901
2902         my $volid = $drive->{file};
2903         return if !$volid;
2904
2905         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2906         if ($storeid) {
2907             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2908             if ($scfg->{disable}) {
2909                 $nodehash = {};
2910             } elsif (my $avail = $scfg->{nodes}) {
2911                 foreach my $node (keys %$nodehash) {
2912                     delete $nodehash->{$node} if !$avail->{$node};
2913                 }
2914             } elsif (!$scfg->{shared}) {
2915                 foreach my $node (keys %$nodehash) {
2916                     delete $nodehash->{$node} if $node ne $nodename
2917                 }
2918             }
2919         }
2920     });
2921
2922     return $nodehash
2923 }
2924
2925 sub check_local_storage_availability {
2926     my ($conf, $storecfg) = @_;
2927
2928     my $nodelist = PVE::Cluster::get_nodelist();
2929     my $nodehash = { map { $_ => {} } @$nodelist };
2930
2931     foreach_drive($conf, sub {
2932         my ($ds, $drive) = @_;
2933
2934         my $volid = $drive->{file};
2935         return if !$volid;
2936
2937         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2938         if ($storeid) {
2939             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2940
2941             if ($scfg->{disable}) {
2942                 foreach my $node (keys %$nodehash) {
2943                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2944                 }
2945             } elsif (my $avail = $scfg->{nodes}) {
2946                 foreach my $node (keys %$nodehash) {
2947                     if (!$avail->{$node}) {
2948                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2949                     }
2950                 }
2951             }
2952         }
2953     });
2954
2955     foreach my $node (values %$nodehash) {
2956         if (my $unavail = $node->{unavailable_storages}) {
2957             $node->{unavailable_storages} = [ sort keys %$unavail ];
2958         }
2959     }
2960
2961     return $nodehash
2962 }
2963
2964 sub check_cmdline {
2965     my ($pidfile, $pid) = @_;
2966
2967     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2968     if (defined($fh)) {
2969         my $line = <$fh>;
2970         $fh->close;
2971         return undef if !$line;
2972         my @param = split(/\0/, $line);
2973
2974         my $cmd = $param[0];
2975         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2976
2977         for (my $i = 0; $i < scalar (@param); $i++) {
2978             my $p = $param[$i];
2979             next if !$p;
2980             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2981                 my $p = $param[$i+1];
2982                 return 1 if $p && ($p eq $pidfile);
2983                 return undef;
2984             }
2985         }
2986     }
2987     return undef;
2988 }
2989
2990 sub check_running {
2991     my ($vmid, $nocheck, $node) = @_;
2992
2993     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2994
2995     die "unable to find configuration file for VM $vmid - no such machine\n"
2996         if !$nocheck && ! -f $filename;
2997
2998     my $pidfile = pidfile_name($vmid);
2999
3000     if (my $fd = IO::File->new("<$pidfile")) {
3001         my $st = stat($fd);
3002         my $line = <$fd>;
3003         close($fd);
3004
3005         my $mtime = $st->mtime;
3006         if ($mtime > time()) {
3007             warn "file '$filename' modified in future\n";
3008         }
3009
3010         if ($line =~ m/^(\d+)$/) {
3011             my $pid = $1;
3012             if (check_cmdline($pidfile, $pid)) {
3013                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3014                     return $pid;
3015                 }
3016             }
3017         }
3018     }
3019
3020     return undef;
3021 }
3022
3023 sub vzlist {
3024
3025     my $vzlist = config_list();
3026
3027     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3028
3029     while (defined(my $de = $fd->read)) {
3030         next if $de !~ m/^(\d+)\.pid$/;
3031         my $vmid = $1;
3032         next if !defined($vzlist->{$vmid});
3033         if (my $pid = check_running($vmid)) {
3034             $vzlist->{$vmid}->{pid} = $pid;
3035         }
3036     }
3037
3038     return $vzlist;
3039 }
3040
3041 sub disksize {
3042     my ($storecfg, $conf) = @_;
3043
3044     my $bootdisk = $conf->{bootdisk};
3045     return undef if !$bootdisk;
3046     return undef if !is_valid_drivename($bootdisk);
3047
3048     return undef if !$conf->{$bootdisk};
3049
3050     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3051     return undef if !defined($drive);
3052
3053     return undef if drive_is_cdrom($drive);
3054
3055     my $volid = $drive->{file};
3056     return undef if !$volid;
3057
3058     return $drive->{size};
3059 }
3060
3061 our $vmstatus_return_properties = {
3062     vmid => get_standard_option('pve-vmid'),
3063     status => {
3064         description => "Qemu process status.",
3065         type => 'string',
3066         enum => ['stopped', 'running'],
3067     },
3068     maxmem => {
3069         description => "Maximum memory in bytes.",
3070         type => 'integer',
3071         optional => 1,
3072         renderer => 'bytes',
3073     },
3074     maxdisk => {
3075         description => "Root disk size in bytes.",
3076         type => 'integer',
3077         optional => 1,
3078         renderer => 'bytes',
3079     },
3080     name => {
3081         description => "VM name.",
3082         type => 'string',
3083         optional => 1,
3084     },
3085     qmpstatus => {
3086         description => "Qemu QMP agent status.",
3087         type => 'string',
3088         optional => 1,
3089     },
3090     pid => {
3091         description => "PID of running qemu process.",
3092         type => 'integer',
3093         optional => 1,
3094     },
3095     uptime => {
3096         description => "Uptime.",
3097         type => 'integer',
3098         optional => 1,
3099         renderer => 'duration',
3100     },
3101     cpus => {
3102         description => "Maximum usable CPUs.",
3103         type => 'number',
3104         optional => 1,
3105     },
3106     lock => {
3107         description => "The current config lock, if any.",
3108         type => 'string',
3109         optional => 1,
3110     }
3111 };
3112
3113 my $last_proc_pid_stat;
3114
3115 # get VM status information
3116 # This must be fast and should not block ($full == false)
3117 # We only query KVM using QMP if $full == true (this can be slow)
3118 sub vmstatus {
3119     my ($opt_vmid, $full) = @_;
3120
3121     my $res = {};
3122
3123     my $storecfg = PVE::Storage::config();
3124
3125     my $list = vzlist();
3126     my $defaults = load_defaults();
3127
3128     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3129
3130     my $cpucount = $cpuinfo->{cpus} || 1;
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134
3135         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3136         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3137
3138         my $d = { vmid => $vmid };
3139         $d->{pid} = $list->{$vmid}->{pid};
3140
3141         # fixme: better status?
3142         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3143
3144         my $size = disksize($storecfg, $conf);
3145         if (defined($size)) {
3146             $d->{disk} = 0; # no info available
3147             $d->{maxdisk} = $size;
3148         } else {
3149             $d->{disk} = 0;
3150             $d->{maxdisk} = 0;
3151         }
3152
3153         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3154             * ($conf->{cores} || $defaults->{cores});
3155         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3156         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3157
3158         $d->{name} = $conf->{name} || "VM $vmid";
3159         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3160             : $defaults->{memory}*(1024*1024);
3161
3162         if ($conf->{balloon}) {
3163             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3164             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3165                 : $defaults->{shares};
3166         }
3167
3168         $d->{uptime} = 0;
3169         $d->{cpu} = 0;
3170         $d->{mem} = 0;
3171
3172         $d->{netout} = 0;
3173         $d->{netin} = 0;
3174
3175         $d->{diskread} = 0;
3176         $d->{diskwrite} = 0;
3177
3178         $d->{template} = PVE::QemuConfig->is_template($conf);
3179
3180         $d->{serial} = 1 if conf_has_serial($conf);
3181         $d->{lock} = $conf->{lock} if $conf->{lock};
3182
3183         $res->{$vmid} = $d;
3184     }
3185
3186     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3187     foreach my $dev (keys %$netdev) {
3188         next if $dev !~ m/^tap([1-9]\d*)i/;
3189         my $vmid = $1;
3190         my $d = $res->{$vmid};
3191         next if !$d;
3192
3193         $d->{netout} += $netdev->{$dev}->{receive};
3194         $d->{netin} += $netdev->{$dev}->{transmit};
3195
3196         if ($full) {
3197             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3198             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3199         }
3200
3201     }
3202
3203     my $ctime = gettimeofday;
3204
3205     foreach my $vmid (keys %$list) {
3206
3207         my $d = $res->{$vmid};
3208         my $pid = $d->{pid};
3209         next if !$pid;
3210
3211         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3212         next if !$pstat; # not running
3213
3214         my $used = $pstat->{utime} + $pstat->{stime};
3215
3216         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3217
3218         if ($pstat->{vsize}) {
3219             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3220         }
3221
3222         my $old = $last_proc_pid_stat->{$pid};
3223         if (!$old) {
3224             $last_proc_pid_stat->{$pid} = {
3225                 time => $ctime,
3226                 used => $used,
3227                 cpu => 0,
3228             };
3229             next;
3230         }
3231
3232         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3233
3234         if ($dtime > 1000) {
3235             my $dutime = $used -  $old->{used};
3236
3237             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3238             $last_proc_pid_stat->{$pid} = {
3239                 time => $ctime,
3240                 used => $used,
3241                 cpu => $d->{cpu},
3242             };
3243         } else {
3244             $d->{cpu} = $old->{cpu};
3245         }
3246     }
3247
3248     return $res if !$full;
3249
3250     my $qmpclient = PVE::QMPClient->new();
3251
3252     my $ballooncb = sub {
3253         my ($vmid, $resp) = @_;
3254
3255         my $info = $resp->{'return'};
3256         return if !$info->{max_mem};
3257
3258         my $d = $res->{$vmid};
3259
3260         # use memory assigned to VM
3261         $d->{maxmem} = $info->{max_mem};
3262         $d->{balloon} = $info->{actual};
3263
3264         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3265             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3266             $d->{freemem} = $info->{free_mem};
3267         }
3268
3269         $d->{ballooninfo} = $info;
3270     };
3271
3272     my $blockstatscb = sub {
3273         my ($vmid, $resp) = @_;
3274         my $data = $resp->{'return'} || [];
3275         my $totalrdbytes = 0;
3276         my $totalwrbytes = 0;
3277
3278         for my $blockstat (@$data) {
3279             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3280             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3281
3282             $blockstat->{device} =~ s/drive-//;
3283             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3284         }
3285         $res->{$vmid}->{diskread} = $totalrdbytes;
3286         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3287     };
3288
3289     my $statuscb = sub {
3290         my ($vmid, $resp) = @_;
3291
3292         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3293         # this fails if ballon driver is not loaded, so this must be
3294         # the last commnand (following command are aborted if this fails).
3295         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3296
3297         my $status = 'unknown';
3298         if (!defined($status = $resp->{'return'}->{status})) {
3299             warn "unable to get VM status\n";
3300             return;
3301         }
3302
3303         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3304     };
3305
3306     foreach my $vmid (keys %$list) {
3307         next if $opt_vmid && ($vmid ne $opt_vmid);
3308         next if !$res->{$vmid}->{pid}; # not running
3309         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3310     }
3311
3312     $qmpclient->queue_execute(undef, 2);
3313
3314     foreach my $vmid (keys %$list) {
3315         next if $opt_vmid && ($vmid ne $opt_vmid);
3316         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3317     }
3318
3319     return $res;
3320 }
3321
3322 sub foreach_drive {
3323     my ($conf, $func, @param) = @_;
3324
3325     foreach my $ds (valid_drive_names()) {
3326         next if !defined($conf->{$ds});
3327
3328         my $drive = parse_drive($ds, $conf->{$ds});
3329         next if !$drive;
3330
3331         &$func($ds, $drive, @param);
3332     }
3333 }
3334
3335 sub foreach_volid {
3336     my ($conf, $func, @param) = @_;
3337
3338     my $volhash = {};
3339
3340     my $test_volid = sub {
3341         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3342
3343         return if !$volid;
3344
3345         $volhash->{$volid}->{cdrom} //= 1;
3346         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3347
3348         $volhash->{$volid}->{replicate} //= 0;
3349         $volhash->{$volid}->{replicate} = 1 if $replicate;
3350
3351         $volhash->{$volid}->{shared} //= 0;
3352         $volhash->{$volid}->{shared} = 1 if $shared;
3353
3354         $volhash->{$volid}->{referenced_in_config} //= 0;
3355         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3356
3357         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3358             if defined($snapname);
3359         $volhash->{$volid}->{size} = $size if $size;
3360     };
3361
3362     foreach_drive($conf, sub {
3363         my ($ds, $drive) = @_;
3364         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3365     });
3366
3367     foreach my $snapname (keys %{$conf->{snapshots}}) {
3368         my $snap = $conf->{snapshots}->{$snapname};
3369         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3370         foreach_drive($snap, sub {
3371             my ($ds, $drive) = @_;
3372             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3373         });
3374     }
3375
3376     foreach my $volid (keys %$volhash) {
3377         &$func($volid, $volhash->{$volid}, @param);
3378     }
3379 }
3380
3381 sub conf_has_serial {
3382     my ($conf) = @_;
3383
3384     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3385         if ($conf->{"serial$i"}) {
3386             return 1;
3387         }
3388     }
3389
3390     return 0;
3391 }
3392
3393 sub conf_has_audio {
3394     my ($conf, $id) = @_;
3395
3396     $id //= 0;
3397     my $audio = $conf->{"audio$id"};
3398     return undef if !defined($audio);
3399
3400     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3401     my $audiodriver = $audioproperties->{driver} // 'spice';
3402
3403     return {
3404         dev => $audioproperties->{device},
3405         dev_id => "audiodev$id",
3406         backend => $audiodriver,
3407         backend_id => "$audiodriver-backend${id}",
3408     };
3409 }
3410
3411 sub vga_conf_has_spice {
3412     my ($vga) = @_;
3413
3414     my $vgaconf = parse_vga($vga);
3415     my $vgatype = $vgaconf->{type};
3416     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3417
3418     return $1 || 1;
3419 }
3420
3421 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3422 sub get_host_arch() {
3423     $host_arch = (POSIX::uname())[4] if !$host_arch;
3424     return $host_arch;
3425 }
3426
3427 sub is_native($) {
3428     my ($arch) = @_;
3429     return get_host_arch() eq $arch;
3430 }
3431
3432 my $default_machines = {
3433     x86_64 => 'pc',
3434     aarch64 => 'virt',
3435 };
3436
3437 sub get_basic_machine_info {
3438     my ($conf, $forcemachine) = @_;
3439
3440     my $arch = $conf->{arch} // get_host_arch();
3441     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3442     return ($arch, $machine);
3443 }
3444
3445 sub get_ovmf_files($) {
3446     my ($arch) = @_;
3447
3448     my $ovmf = $OVMF->{$arch}
3449         or die "no OVMF images known for architecture '$arch'\n";
3450
3451     return @$ovmf;
3452 }
3453
3454 my $Arch2Qemu = {
3455     aarch64 => '/usr/bin/qemu-system-aarch64',
3456     x86_64 => '/usr/bin/qemu-system-x86_64',
3457 };
3458 sub get_command_for_arch($) {
3459     my ($arch) = @_;
3460     return '/usr/bin/kvm' if is_native($arch);
3461
3462     my $cmd = $Arch2Qemu->{$arch}
3463         or die "don't know how to emulate architecture '$arch'\n";
3464     return $cmd;
3465 }
3466
3467 sub get_cpu_options {
3468     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3469
3470     my $cpuFlags = [];
3471     my $ostype = $conf->{ostype};
3472
3473     my $cpu = $kvm ? "kvm64" : "qemu64";
3474     if ($arch eq 'aarch64') {
3475         $cpu = 'cortex-a57';
3476     }
3477     my $hv_vendor_id;
3478     if (my $cputype = $conf->{cpu}) {
3479         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3480             or die "Cannot parse cpu description: $cputype\n";
3481         $cpu = $cpuconf->{cputype};
3482         $kvm_off = 1 if $cpuconf->{hidden};
3483         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3484
3485         if (defined(my $flags = $cpuconf->{flags})) {
3486             push @$cpuFlags, split(";", $flags);
3487         }
3488     }
3489
3490     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3491
3492     push @$cpuFlags , '-x2apic'
3493         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3494
3495     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3496
3497     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3498
3499     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3500
3501         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3502         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3503     }
3504
3505     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3506
3507     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3508
3509     push @$cpuFlags, 'kvm=off' if $kvm_off;
3510
3511     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3512         push @$cpuFlags, "vendor=${cpu_vendor}"
3513             if $cpu_vendor ne 'default';
3514     } elsif ($arch ne 'aarch64') {
3515         die "internal error"; # should not happen
3516     }
3517
3518     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3519
3520     return ('-cpu', $cpu);
3521 }
3522
3523 sub config_to_command {
3524     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3525
3526     my $cmd = [];
3527     my $globalFlags = [];
3528     my $machineFlags = [];
3529     my $rtcFlags = [];
3530     my $devices = [];
3531     my $pciaddr = '';
3532     my $bridges = {};
3533     my $vernum = 0; # unknown
3534     my $ostype = $conf->{ostype};
3535     my $winversion = windows_version($ostype);
3536     my $kvm = $conf->{kvm};
3537
3538     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3539     my $kvm_binary = get_command_for_arch($arch);
3540     my $kvmver = kvm_user_version($kvm_binary);
3541     $kvm //= 1 if is_native($arch);
3542
3543     if ($kvm) {
3544         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3545             if !defined kvm_version();
3546     }
3547
3548     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3549         $vernum = $1*1000000+$2*1000;
3550     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3551         $vernum = $1*1000000+$2*1000+$3;
3552     }
3553
3554     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3555
3556     my $q35 = machine_type_is_q35($conf);
3557     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3558     my $use_old_bios_files = undef;
3559     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3560
3561     my $cpuunits = defined($conf->{cpuunits}) ?
3562             $conf->{cpuunits} : $defaults->{cpuunits};
3563
3564     push @$cmd, $kvm_binary;
3565
3566     push @$cmd, '-id', $vmid;
3567
3568     my $vmname = $conf->{name} || "vm$vmid";
3569
3570     push @$cmd, '-name', $vmname;
3571
3572     my $use_virtio = 0;
3573
3574     my $qmpsocket = qmp_socket($vmid);
3575     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3576     push @$cmd, '-mon', "chardev=qmp,mode=control";
3577
3578     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3579         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3580         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3581     }
3582
3583     push @$cmd, '-pidfile' , pidfile_name($vmid);
3584
3585     push @$cmd, '-daemonize';
3586
3587     if ($conf->{smbios1}) {
3588         my $smbios_conf = parse_smbios1($conf->{smbios1});
3589         if ($smbios_conf->{base64}) {
3590             # Do not pass base64 flag to qemu
3591             delete $smbios_conf->{base64};
3592             my $smbios_string = "";
3593             foreach my $key (keys %$smbios_conf) {
3594                 my $value;
3595                 if ($key eq "uuid") {
3596                     $value = $smbios_conf->{uuid}
3597                 } else {
3598                     $value = decode_base64($smbios_conf->{$key});
3599                 }
3600                 # qemu accepts any binary data, only commas need escaping by double comma
3601                 $value =~ s/,/,,/g;
3602                 $smbios_string .= "," . $key . "=" . $value if $value;
3603             }
3604             push @$cmd, '-smbios', "type=1" . $smbios_string;
3605         } else {
3606             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3607         }
3608     }
3609
3610     if ($conf->{vmgenid}) {
3611         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3612     }
3613
3614     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3615     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3616         die "uefi base image not found\n" if ! -f $ovmf_code;
3617
3618         my $path;
3619         my $format;
3620         if (my $efidisk = $conf->{efidisk0}) {
3621             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3622             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3623             $format = $d->{format};
3624             if ($storeid) {
3625                 $path = PVE::Storage::path($storecfg, $d->{file});
3626                 if (!defined($format)) {
3627                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3628                     $format = qemu_img_format($scfg, $volname);
3629                 }
3630             } else {
3631                 $path = $d->{file};
3632                 die "efidisk format must be specified\n"
3633                     if !defined($format);
3634             }
3635         } else {
3636             warn "no efidisk configured! Using temporary efivars disk.\n";
3637             $path = "/tmp/$vmid-ovmf.fd";
3638             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3639             $format = 'raw';
3640         }
3641
3642         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3643         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3644     }
3645
3646     # load q35 config
3647     if ($q35) {
3648         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3649         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3650             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3651         } else {
3652             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3653         }
3654     }
3655
3656     # add usb controllers
3657     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3658     push @$devices, @usbcontrollers if @usbcontrollers;
3659     my $vga = parse_vga($conf->{vga});
3660
3661     my $qxlnum = vga_conf_has_spice($conf->{vga});
3662     $vga->{type} = 'qxl' if $qxlnum;
3663
3664     if (!$vga->{type}) {
3665         if ($arch eq 'aarch64') {
3666             $vga->{type} = 'virtio';
3667         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3668             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3669         } else {
3670             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3671         }
3672     }
3673
3674     # enable absolute mouse coordinates (needed by vnc)
3675     my $tablet;
3676     if (defined($conf->{tablet})) {
3677         $tablet = $conf->{tablet};
3678     } else {
3679         $tablet = $defaults->{tablet};
3680         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3681         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3682     }
3683
3684     if ($tablet) {
3685         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3686         my $kbd = print_keyboarddevice_full($conf, $arch);
3687         push @$devices, '-device', $kbd if defined($kbd);
3688     }
3689
3690     my $kvm_off = 0;
3691     my $gpu_passthrough;
3692
3693     # host pci devices
3694     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3695         my $id = "hostpci$i";
3696         my $d = parse_hostpci($conf->{$id});
3697         next if !$d;
3698
3699         if (my $pcie = $d->{pcie}) {
3700             die "q35 machine model is not enabled" if !$q35;
3701             # win7 wants to have the pcie devices directly on the pcie bus
3702             # instead of in the root port
3703             if ($winversion == 7) {
3704                 $pciaddr = print_pcie_addr("${id}bus0");
3705             } else {
3706                 # add more root ports if needed, 4 are present by default
3707                 # by pve-q35 cfgs, rest added here on demand.
3708                 if ($i > 3) {
3709                     push @$devices, '-device', print_pcie_root_port($i);
3710                 }
3711                 $pciaddr = print_pcie_addr($id);
3712             }
3713         } else {
3714             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3715         }
3716
3717         my $xvga = '';
3718         if ($d->{'x-vga'}) {
3719             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3720             $kvm_off = 1;
3721             $vga->{type} = 'none' if !defined($conf->{vga});
3722             $gpu_passthrough = 1;
3723         }
3724
3725         my $pcidevices = $d->{pciid};
3726         my $multifunction = 1 if @$pcidevices > 1;
3727
3728         my $sysfspath;
3729         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3730             my $pci_id = $pcidevices->[0]->{id};
3731             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3732             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3733         } elsif ($d->{mdev}) {
3734             warn "ignoring mediated device '$id' with multifunction device\n";
3735         }
3736
3737         my $j=0;
3738         foreach my $pcidevice (@$pcidevices) {
3739             my $devicestr = "vfio-pci";
3740
3741             if ($sysfspath) {
3742                 $devicestr .= ",sysfsdev=$sysfspath";
3743             } else {
3744                 $devicestr .= ",host=$pcidevice->{id}";
3745             }
3746
3747             my $mf_addr = $multifunction ? ".$j" : '';
3748             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3749
3750             if ($j == 0) {
3751                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3752                 $devicestr .= "$xvga";
3753                 $devicestr .= ",multifunction=on" if $multifunction;
3754                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3755             }
3756
3757             push @$devices, '-device', $devicestr;
3758             $j++;
3759         }
3760     }
3761
3762     # usb devices
3763     my $usb_dev_features = {};
3764     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3765
3766     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3767     push @$devices, @usbdevices if @usbdevices;
3768     # serial devices
3769     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3770         if (my $path = $conf->{"serial$i"}) {
3771             if ($path eq 'socket') {
3772                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3773                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3774                 # On aarch64, serial0 is the UART device. Qemu only allows
3775                 # connecting UART devices via the '-serial' command line, as
3776                 # the device has a fixed slot on the hardware...
3777                 if ($arch eq 'aarch64' && $i == 0) {
3778                     push @$devices, '-serial', "chardev:serial$i";
3779                 } else {
3780                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3781                 }
3782             } else {
3783                 die "no such serial device\n" if ! -c $path;
3784                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3785                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3786             }
3787         }
3788     }
3789
3790     # parallel devices
3791     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3792         if (my $path = $conf->{"parallel$i"}) {
3793             die "no such parallel device\n" if ! -c $path;
3794             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3795             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3796             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3797         }
3798     }
3799
3800     if (my $audio = conf_has_audio($conf)) {
3801
3802         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3803
3804         my $id = $audio->{dev_id};
3805         if ($audio->{dev} eq 'AC97') {
3806             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3807         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3808             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3809             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3810             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3811         } else {
3812             die "unkown audio device '$audio->{dev}', implement me!";
3813         }
3814
3815         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3816     }
3817
3818     my $sockets = 1;
3819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3820     $sockets = $conf->{sockets} if  $conf->{sockets};
3821
3822     my $cores = $conf->{cores} || 1;
3823
3824     my $maxcpus = $sockets * $cores;
3825
3826     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3827
3828     my $allowed_vcpus = $cpuinfo->{cpus};
3829
3830     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3831         if ($allowed_vcpus < $maxcpus);
3832
3833     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3834
3835         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3836         for (my $i = 2; $i <= $vcpus; $i++)  {
3837             my $cpustr = print_cpu_device($conf,$i);
3838             push @$cmd, '-device', $cpustr;
3839         }
3840
3841     } else {
3842
3843         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3844     }
3845     push @$cmd, '-nodefaults';
3846
3847     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3848
3849     my $bootindex_hash = {};
3850     my $i = 1;
3851     foreach my $o (split(//, $bootorder)) {
3852         $bootindex_hash->{$o} = $i*100;
3853         $i++;
3854     }
3855
3856     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3857
3858     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3859
3860     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3861
3862     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3863         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3864         my $socket = vnc_socket($vmid);
3865         push @$cmd,  '-vnc', "unix:$socket,password";
3866     } else {
3867         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3868         push @$cmd, '-nographic';
3869     }
3870
3871     # time drift fix
3872     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3873
3874     my $useLocaltime = $conf->{localtime};
3875
3876     if ($winversion >= 5) { # windows
3877         $useLocaltime = 1 if !defined($conf->{localtime});
3878
3879         # use time drift fix when acpi is enabled
3880         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3881             $tdf = 1 if !defined($conf->{tdf});
3882         }
3883     }
3884
3885     if ($winversion >= 6) {
3886         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3887         push @$cmd, '-no-hpet';
3888     }
3889
3890     push @$rtcFlags, 'driftfix=slew' if $tdf;
3891
3892     if (!$kvm) {
3893         push @$machineFlags, 'accel=tcg';
3894     }
3895
3896     if ($machine_type) {
3897         push @$machineFlags, "type=${machine_type}";
3898     }
3899
3900     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3901         push @$rtcFlags, "base=$conf->{startdate}";
3902     } elsif ($useLocaltime) {
3903         push @$rtcFlags, 'base=localtime';
3904     }
3905
3906     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3907
3908     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3909
3910     push @$cmd, '-S' if $conf->{freeze};
3911
3912     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3913
3914     if (parse_guest_agent($conf)->{enabled}) {
3915         my $qgasocket = qmp_socket($vmid, 1);
3916         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3917         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3918         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3919         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3920     }
3921
3922     my $spice_port;
3923
3924     if ($qxlnum) {
3925         if ($qxlnum > 1) {
3926             if ($winversion){
3927                 for(my $i = 1; $i < $qxlnum; $i++){
3928                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3929                 }
3930             } else {
3931                 # assume other OS works like Linux
3932                 my ($ram, $vram) = ("134217728", "67108864");
3933                 if ($vga->{memory}) {
3934                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3935                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3936                 }
3937                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3938                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3939             }
3940         }
3941
3942         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3943
3944         my $nodename = PVE::INotify::nodename();
3945         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3946         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3947         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3948
3949         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3950         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3951         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3952
3953         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3954         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3955
3956         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3957         if ($spice_enhancement->{foldersharing}) {
3958             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3959             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3960         }
3961
3962         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3963         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3964         push @$devices, '-spice', "$spice_opts";
3965     }
3966
3967     # enable balloon by default, unless explicitly disabled
3968     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3969         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3970         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3971     }
3972
3973     if ($conf->{watchdog}) {
3974         my $wdopts = parse_watchdog($conf->{watchdog});
3975         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3976         my $watchdog = $wdopts->{model} || 'i6300esb';
3977         push @$devices, '-device', "$watchdog$pciaddr";
3978         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3979     }
3980
3981     my $vollist = [];
3982     my $scsicontroller = {};
3983     my $ahcicontroller = {};
3984     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3985
3986     # Add iscsi initiator name if available
3987     if (my $initiator = get_initiator_name()) {
3988         push @$devices, '-iscsi', "initiator-name=$initiator";
3989     }
3990
3991     foreach_drive($conf, sub {
3992         my ($ds, $drive) = @_;
3993
3994         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3995             push @$vollist, $drive->{file};
3996         }
3997
3998         # ignore efidisk here, already added in bios/fw handling code above
3999         return if $drive->{interface} eq 'efidisk';
4000
4001         $use_virtio = 1 if $ds =~ m/^virtio/;
4002
4003         if (drive_is_cdrom ($drive)) {
4004             if ($bootindex_hash->{d}) {
4005                 $drive->{bootindex} = $bootindex_hash->{d};
4006                 $bootindex_hash->{d} += 1;
4007             }
4008         } else {
4009             if ($bootindex_hash->{c}) {
4010                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4011                 $bootindex_hash->{c} += 1;
4012             }
4013         }
4014
4015         if($drive->{interface} eq 'virtio'){
4016            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4017         }
4018
4019         if ($drive->{interface} eq 'scsi') {
4020
4021             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4022
4023             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4024             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4025
4026             my $iothread = '';
4027             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4028                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4029                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4030             } elsif ($drive->{iothread}) {
4031                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4032             }
4033
4034             my $queues = '';
4035             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4036                 $queues = ",num_queues=$drive->{queues}";
4037             }
4038
4039             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4040             $scsicontroller->{$controller}=1;
4041         }
4042
4043         if ($drive->{interface} eq 'sata') {
4044            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4045            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4046            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4047            $ahcicontroller->{$controller}=1;
4048         }
4049
4050         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4051         push @$devices, '-drive',$drive_cmd;
4052         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4053     });
4054
4055     for (my $i = 0; $i < $MAX_NETS; $i++) {
4056          next if !$conf->{"net$i"};
4057          my $d = parse_net($conf->{"net$i"});
4058          next if !$d;
4059
4060          $use_virtio = 1 if $d->{model} eq 'virtio';
4061
4062          if ($bootindex_hash->{n}) {
4063             $d->{bootindex} = $bootindex_hash->{n};
4064             $bootindex_hash->{n} += 1;
4065          }
4066
4067          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4068          push @$devices, '-netdev', $netdevfull;
4069
4070          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4071          push @$devices, '-device', $netdevicefull;
4072     }
4073
4074     if ($conf->{ivshmem}) {
4075         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4076
4077         my $bus;
4078         if ($q35) {
4079             $bus = print_pcie_addr("ivshmem");
4080         } else {
4081             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4082         }
4083
4084         my $ivshmem_name = $ivshmem->{name} // $vmid;
4085         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4086
4087         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4088         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4089     }
4090
4091     if (!$q35) {
4092         # add pci bridges
4093         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4094            $bridges->{1} = 1;
4095            $bridges->{2} = 1;
4096         }
4097
4098         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4099
4100         for my $k (sort {$b cmp $a} keys %$bridges) {
4101             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4102             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4103         }
4104     }
4105
4106     push @$cmd, @$devices;
4107     push @$cmd, '-rtc', join(',', @$rtcFlags)
4108         if scalar(@$rtcFlags);
4109     push @$cmd, '-machine', join(',', @$machineFlags)
4110         if scalar(@$machineFlags);
4111     push @$cmd, '-global', join(',', @$globalFlags)
4112         if scalar(@$globalFlags);
4113
4114     if (my $vmstate = $conf->{vmstate}) {
4115         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4116         push @$vollist, $vmstate;
4117         push @$cmd, '-loadstate', $statepath;
4118     }
4119
4120     # add custom args
4121     if ($conf->{args}) {
4122         my $aa = PVE::Tools::split_args($conf->{args});
4123         push @$cmd, @$aa;
4124     }
4125
4126     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4127 }
4128
4129 sub vnc_socket {
4130     my ($vmid) = @_;
4131     return "${var_run_tmpdir}/$vmid.vnc";
4132 }
4133
4134 sub spice_port {
4135     my ($vmid) = @_;
4136
4137     my $res = vm_mon_cmd($vmid, 'query-spice');
4138
4139     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4140 }
4141
4142 sub qmp_socket {
4143     my ($vmid, $qga, $name) = @_;
4144     my $sockettype = $qga ? 'qga' : 'qmp';
4145     my $ext = $name ? '-'.$name : '';
4146     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4147 }
4148
4149 sub pidfile_name {
4150     my ($vmid) = @_;
4151     return "${var_run_tmpdir}/$vmid.pid";
4152 }
4153
4154 sub vm_devices_list {
4155     my ($vmid) = @_;
4156
4157     my $res = vm_mon_cmd($vmid, 'query-pci');
4158     my $devices_to_check = [];
4159     my $devices = {};
4160     foreach my $pcibus (@$res) {
4161         push @$devices_to_check, @{$pcibus->{devices}},
4162     }
4163
4164     while (@$devices_to_check) {
4165         my $to_check = [];
4166         for my $d (@$devices_to_check) {
4167             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4168             next if !$d->{'pci_bridge'};
4169
4170             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4171             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4172         }
4173         $devices_to_check = $to_check;
4174     }
4175
4176     my $resblock = vm_mon_cmd($vmid, 'query-block');
4177     foreach my $block (@$resblock) {
4178         if($block->{device} =~ m/^drive-(\S+)/){
4179                 $devices->{$1} = 1;
4180         }
4181     }
4182
4183     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4184     foreach my $mice (@$resmice) {
4185         if ($mice->{name} eq 'QEMU HID Tablet') {
4186             $devices->{tablet} = 1;
4187             last;
4188         }
4189     }
4190
4191     # for usb devices there is no query-usb
4192     # but we can iterate over the entries in
4193     # qom-list path=/machine/peripheral
4194     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4195     foreach my $per (@$resperipheral) {
4196         if ($per->{name} =~ m/^usb\d+$/) {
4197             $devices->{$per->{name}} = 1;
4198         }
4199     }
4200
4201     return $devices;
4202 }
4203
4204 sub vm_deviceplug {
4205     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4206
4207     my $q35 = machine_type_is_q35($conf);
4208
4209     my $devices_list = vm_devices_list($vmid);
4210     return 1 if defined($devices_list->{$deviceid});
4211
4212     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4213
4214     if ($deviceid eq 'tablet') {
4215
4216         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4217
4218     } elsif ($deviceid eq 'keyboard') {
4219
4220         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4221
4222     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4223
4224         die "usb hotplug currently not reliable\n";
4225         # since we can't reliably hot unplug all added usb devices
4226         # and usb passthrough disables live migration
4227         # we disable usb hotplugging for now
4228         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4229
4230     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4231
4232         qemu_iothread_add($vmid, $deviceid, $device);
4233
4234         qemu_driveadd($storecfg, $vmid, $device);
4235         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         eval { qemu_deviceaddverify($vmid, $deviceid); };
4239         if (my $err = $@) {
4240             eval { qemu_drivedel($vmid, $deviceid); };
4241             warn $@ if $@;
4242             die $err;
4243         }
4244
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246
4247
4248         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4249         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4250         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4251
4252         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4253
4254         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4255             qemu_iothread_add($vmid, $deviceid, $device);
4256             $devicefull .= ",iothread=iothread-$deviceid";
4257         }
4258
4259         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4260             $devicefull .= ",num_queues=$device->{queues}";
4261         }
4262
4263         qemu_deviceadd($vmid, $devicefull);
4264         qemu_deviceaddverify($vmid, $deviceid);
4265
4266     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4267
4268         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4269         qemu_driveadd($storecfg, $vmid, $device);
4270
4271         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4272         eval { qemu_deviceadd($vmid, $devicefull); };
4273         if (my $err = $@) {
4274             eval { qemu_drivedel($vmid, $deviceid); };
4275             warn $@ if $@;
4276             die $err;
4277         }
4278
4279     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4280
4281         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4282
4283         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4284         my $use_old_bios_files = undef;
4285         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4286
4287         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4288         qemu_deviceadd($vmid, $netdevicefull);
4289         eval {
4290             qemu_deviceaddverify($vmid, $deviceid);
4291             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4292         };
4293         if (my $err = $@) {
4294             eval { qemu_netdevdel($vmid, $deviceid); };
4295             warn $@ if $@;
4296             die $err;
4297         }
4298
4299     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4300
4301         my $bridgeid = $2;
4302         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4303         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4304
4305         qemu_deviceadd($vmid, $devicefull);
4306         qemu_deviceaddverify($vmid, $deviceid);
4307
4308     } else {
4309         die "can't hotplug device '$deviceid'\n";
4310     }
4311
4312     return 1;
4313 }
4314
4315 # fixme: this should raise exceptions on error!
4316 sub vm_deviceunplug {
4317     my ($vmid, $conf, $deviceid) = @_;
4318
4319     my $devices_list = vm_devices_list($vmid);
4320     return 1 if !defined($devices_list->{$deviceid});
4321
4322     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4323
4324     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4325
4326         qemu_devicedel($vmid, $deviceid);
4327
4328     } elsif ($deviceid =~ m/^usb\d+$/) {
4329
4330         die "usb hotplug currently not reliable\n";
4331         # when unplugging usb devices this way,
4332         # there may be remaining usb controllers/hubs
4333         # so we disable it for now
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336
4337     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4338
4339         qemu_devicedel($vmid, $deviceid);
4340         qemu_devicedelverify($vmid, $deviceid);
4341         qemu_drivedel($vmid, $deviceid);
4342         qemu_iothread_del($conf, $vmid, $deviceid);
4343
4344     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4345
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348         qemu_iothread_del($conf, $vmid, $deviceid);
4349
4350     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4351
4352         qemu_devicedel($vmid, $deviceid);
4353         qemu_drivedel($vmid, $deviceid);
4354         qemu_deletescsihw($conf, $vmid, $deviceid);
4355
4356     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4357
4358         qemu_devicedel($vmid, $deviceid);
4359         qemu_devicedelverify($vmid, $deviceid);
4360         qemu_netdevdel($vmid, $deviceid);
4361
4362     } else {
4363         die "can't unplug device '$deviceid'\n";
4364     }
4365
4366     return 1;
4367 }
4368
4369 sub qemu_deviceadd {
4370     my ($vmid, $devicefull) = @_;
4371
4372     $devicefull = "driver=".$devicefull;
4373     my %options =  split(/[=,]/, $devicefull);
4374
4375     vm_mon_cmd($vmid, "device_add" , %options);
4376 }
4377
4378 sub qemu_devicedel {
4379     my ($vmid, $deviceid) = @_;
4380
4381     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4382 }
4383
4384 sub qemu_iothread_add {
4385     my($vmid, $deviceid, $device) = @_;
4386
4387     if ($device->{iothread}) {
4388         my $iothreads = vm_iothreads_list($vmid);
4389         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4390     }
4391 }
4392
4393 sub qemu_iothread_del {
4394     my($conf, $vmid, $deviceid) = @_;
4395
4396     my $confid = $deviceid;
4397     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4398         $confid = 'scsi' . $1;
4399     }
4400     my $device = parse_drive($confid, $conf->{$confid});
4401     if ($device->{iothread}) {
4402         my $iothreads = vm_iothreads_list($vmid);
4403         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4404     }
4405 }
4406
4407 sub qemu_objectadd {
4408     my($vmid, $objectid, $qomtype) = @_;
4409
4410     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4411
4412     return 1;
4413 }
4414
4415 sub qemu_objectdel {
4416     my($vmid, $objectid) = @_;
4417
4418     vm_mon_cmd($vmid, "object-del", id => $objectid);
4419
4420     return 1;
4421 }
4422
4423 sub qemu_driveadd {
4424     my ($storecfg, $vmid, $device) = @_;
4425
4426     my $drive = print_drive_full($storecfg, $vmid, $device);
4427     $drive =~ s/\\/\\\\/g;
4428     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4429
4430     # If the command succeeds qemu prints: "OK"
4431     return 1 if $ret =~ m/OK/s;
4432
4433     die "adding drive failed: $ret\n";
4434 }
4435
4436 sub qemu_drivedel {
4437     my($vmid, $deviceid) = @_;
4438
4439     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4440     $ret =~ s/^\s+//;
4441
4442     return 1 if $ret eq "";
4443
4444     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4445     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4446
4447     die "deleting drive $deviceid failed : $ret\n";
4448 }
4449
4450 sub qemu_deviceaddverify {
4451     my ($vmid, $deviceid) = @_;
4452
4453     for (my $i = 0; $i <= 5; $i++) {
4454          my $devices_list = vm_devices_list($vmid);
4455          return 1 if defined($devices_list->{$deviceid});
4456          sleep 1;
4457     }
4458
4459     die "error on hotplug device '$deviceid'\n";
4460 }
4461
4462
4463 sub qemu_devicedelverify {
4464     my ($vmid, $deviceid) = @_;
4465
4466     # need to verify that the device is correctly removed as device_del
4467     # is async and empty return is not reliable
4468
4469     for (my $i = 0; $i <= 5; $i++) {
4470          my $devices_list = vm_devices_list($vmid);
4471          return 1 if !defined($devices_list->{$deviceid});
4472          sleep 1;
4473     }
4474
4475     die "error on hot-unplugging device '$deviceid'\n";
4476 }
4477
4478 sub qemu_findorcreatescsihw {
4479     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4480
4481     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4482
4483     my $scsihwid="$controller_prefix$controller";
4484     my $devices_list = vm_devices_list($vmid);
4485
4486     if(!defined($devices_list->{$scsihwid})) {
4487         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4488     }
4489
4490     return 1;
4491 }
4492
4493 sub qemu_deletescsihw {
4494     my ($conf, $vmid, $opt) = @_;
4495
4496     my $device = parse_drive($opt, $conf->{$opt});
4497
4498     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4499         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4500         return 1;
4501     }
4502
4503     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4504
4505     my $devices_list = vm_devices_list($vmid);
4506     foreach my $opt (keys %{$devices_list}) {
4507         if (PVE::QemuServer::is_valid_drivename($opt)) {
4508             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4509             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4510                 return 1;
4511             }
4512         }
4513     }
4514
4515     my $scsihwid="scsihw$controller";
4516
4517     vm_deviceunplug($vmid, $conf, $scsihwid);
4518
4519     return 1;
4520 }
4521
4522 sub qemu_add_pci_bridge {
4523     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4524
4525     my $bridges = {};
4526
4527     my $bridgeid;
4528
4529     print_pci_addr($device, $bridges, $arch, $machine_type);
4530
4531     while (my ($k, $v) = each %$bridges) {
4532         $bridgeid = $k;
4533     }
4534     return 1 if !defined($bridgeid) || $bridgeid < 1;
4535
4536     my $bridge = "pci.$bridgeid";
4537     my $devices_list = vm_devices_list($vmid);
4538
4539     if (!defined($devices_list->{$bridge})) {
4540         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4541     }
4542
4543     return 1;
4544 }
4545
4546 sub qemu_set_link_status {
4547     my ($vmid, $device, $up) = @_;
4548
4549     vm_mon_cmd($vmid, "set_link", name => $device,
4550                up => $up ? JSON::true : JSON::false);
4551 }
4552
4553 sub qemu_netdevadd {
4554     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4555
4556     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4557     my %options =  split(/[=,]/, $netdev);
4558
4559     vm_mon_cmd($vmid, "netdev_add",  %options);
4560     return 1;
4561 }
4562
4563 sub qemu_netdevdel {
4564     my ($vmid, $deviceid) = @_;
4565
4566     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4567 }
4568
4569 sub qemu_usb_hotplug {
4570     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4571
4572     return if !$device;
4573
4574     # remove the old one first
4575     vm_deviceunplug($vmid, $conf, $deviceid);
4576
4577     # check if xhci controller is necessary and available
4578     if ($device->{usb3}) {
4579
4580         my $devicelist = vm_devices_list($vmid);
4581
4582         if (!$devicelist->{xhci}) {
4583             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4584             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4585         }
4586     }
4587     my $d = parse_usb_device($device->{host});
4588     $d->{usb3} = $device->{usb3};
4589
4590     # add the new one
4591     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4592 }
4593
4594 sub qemu_cpu_hotplug {
4595     my ($vmid, $conf, $vcpus) = @_;
4596
4597     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4598
4599     my $sockets = 1;
4600     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4601     $sockets = $conf->{sockets} if  $conf->{sockets};
4602     my $cores = $conf->{cores} || 1;
4603     my $maxcpus = $sockets * $cores;
4604
4605     $vcpus = $maxcpus if !$vcpus;
4606
4607     die "you can't add more vcpus than maxcpus\n"
4608         if $vcpus > $maxcpus;
4609
4610     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4611
4612     if ($vcpus < $currentvcpus) {
4613
4614         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4615
4616             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4617                 qemu_devicedel($vmid, "cpu$i");
4618                 my $retry = 0;
4619                 my $currentrunningvcpus = undef;
4620                 while (1) {
4621                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4622                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4623                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4624                     $retry++;
4625                     sleep 1;
4626                 }
4627                 #update conf after each succesfull cpu unplug
4628                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4629                 PVE::QemuConfig->write_config($vmid, $conf);
4630             }
4631         } else {
4632             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4633         }
4634
4635         return;
4636     }
4637
4638     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4639     die "vcpus in running vm does not match its configuration\n"
4640         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4641
4642     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4643
4644         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4645             my $cpustr = print_cpu_device($conf, $i);
4646             qemu_deviceadd($vmid, $cpustr);
4647
4648             my $retry = 0;
4649             my $currentrunningvcpus = undef;
4650             while (1) {
4651                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4652                 last if scalar(@{$currentrunningvcpus}) == $i;
4653                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4654                 sleep 1;
4655                 $retry++;
4656             }
4657             #update conf after each succesfull cpu hotplug
4658             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4659             PVE::QemuConfig->write_config($vmid, $conf);
4660         }
4661     } else {
4662
4663         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4664             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4665         }
4666     }
4667 }
4668
4669 sub qemu_block_set_io_throttle {
4670     my ($vmid, $deviceid,
4671         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4672         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4673         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4674         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4675
4676     return if !check_running($vmid) ;
4677
4678     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4679         bps => int($bps),
4680         bps_rd => int($bps_rd),
4681         bps_wr => int($bps_wr),
4682         iops => int($iops),
4683         iops_rd => int($iops_rd),
4684         iops_wr => int($iops_wr),
4685         bps_max => int($bps_max),
4686         bps_rd_max => int($bps_rd_max),
4687         bps_wr_max => int($bps_wr_max),
4688         iops_max => int($iops_max),
4689         iops_rd_max => int($iops_rd_max),
4690         iops_wr_max => int($iops_wr_max),
4691         bps_max_length => int($bps_max_length),
4692         bps_rd_max_length => int($bps_rd_max_length),
4693         bps_wr_max_length => int($bps_wr_max_length),
4694         iops_max_length => int($iops_max_length),
4695         iops_rd_max_length => int($iops_rd_max_length),
4696         iops_wr_max_length => int($iops_wr_max_length),
4697     );
4698
4699 }
4700
4701 # old code, only used to shutdown old VM after update
4702 sub __read_avail {
4703     my ($fh, $timeout) = @_;
4704
4705     my $sel = new IO::Select;
4706     $sel->add($fh);
4707
4708     my $res = '';
4709     my $buf;
4710
4711     my @ready;
4712     while (scalar (@ready = $sel->can_read($timeout))) {
4713         my $count;
4714         if ($count = $fh->sysread($buf, 8192)) {
4715             if ($buf =~ /^(.*)\(qemu\) $/s) {
4716                 $res .= $1;
4717                 last;
4718             } else {
4719                 $res .= $buf;
4720             }
4721         } else {
4722             if (!defined($count)) {
4723                 die "$!\n";
4724             }
4725             last;
4726         }
4727     }
4728
4729     die "monitor read timeout\n" if !scalar(@ready);
4730
4731     return $res;
4732 }
4733
4734 sub qemu_block_resize {
4735     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4736
4737     my $running = check_running($vmid);
4738
4739     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4740
4741     return if !$running;
4742
4743     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4744
4745 }
4746
4747 sub qemu_volume_snapshot {
4748     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4749
4750     my $running = check_running($vmid);
4751
4752     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4753         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4754     } else {
4755         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4756     }
4757 }
4758
4759 sub qemu_volume_snapshot_delete {
4760     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4761
4762     my $running = check_running($vmid);
4763
4764     if($running) {
4765
4766         $running = undef;
4767         my $conf = PVE::QemuConfig->load_config($vmid);
4768         foreach_drive($conf, sub {
4769             my ($ds, $drive) = @_;
4770             $running = 1 if $drive->{file} eq $volid;
4771         });
4772     }
4773
4774     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4775         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4776     } else {
4777         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4778     }
4779 }
4780
4781 sub set_migration_caps {
4782     my ($vmid) = @_;
4783
4784     my $cap_ref = [];
4785
4786     my $enabled_cap = {
4787         "auto-converge" => 1,
4788         "xbzrle" => 1,
4789         "x-rdma-pin-all" => 0,
4790         "zero-blocks" => 0,
4791         "compress" => 0
4792     };
4793
4794     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4795
4796     for my $supported_capability (@$supported_capabilities) {
4797         push @$cap_ref, {
4798             capability => $supported_capability->{capability},
4799             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4800         };
4801     }
4802
4803     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4804 }
4805
4806 my $fast_plug_option = {
4807     'lock' => 1,
4808     'name' => 1,
4809     'onboot' => 1,
4810     'shares' => 1,
4811     'startup' => 1,
4812     'description' => 1,
4813     'protection' => 1,
4814     'vmstatestorage' => 1,
4815     'hookscript' => 1,
4816 };
4817
4818 # hotplug changes in [PENDING]
4819 # $selection hash can be used to only apply specified options, for
4820 # example: { cores => 1 } (only apply changed 'cores')
4821 # $errors ref is used to return error messages
4822 sub vmconfig_hotplug_pending {
4823     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4824
4825     my $defaults = load_defaults();
4826     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4827
4828     # commit values which do not have any impact on running VM first
4829     # Note: those option cannot raise errors, we we do not care about
4830     # $selection and always apply them.
4831
4832     my $add_error = sub {
4833         my ($opt, $msg) = @_;
4834         $errors->{$opt} = "hotplug problem - $msg";
4835     };
4836
4837     my $changes = 0;
4838     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4839         if ($fast_plug_option->{$opt}) {
4840             $conf->{$opt} = $conf->{pending}->{$opt};
4841             delete $conf->{pending}->{$opt};
4842             $changes = 1;
4843         }
4844     }
4845
4846     if ($changes) {
4847         PVE::QemuConfig->write_config($vmid, $conf);
4848         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4849     }
4850
4851     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4852
4853     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4854     foreach my $opt (sort keys %$pending_delete_hash) {
4855         next if $selection && !$selection->{$opt};
4856         my $force = $pending_delete_hash->{$opt}->{force};
4857         eval {
4858             if ($opt eq 'hotplug') {
4859                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4860             } elsif ($opt eq 'tablet') {
4861                 die "skip\n" if !$hotplug_features->{usb};
4862                 if ($defaults->{tablet}) {
4863                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4864                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4865                         if $arch eq 'aarch64';
4866                 } else {
4867                     vm_deviceunplug($vmid, $conf, 'tablet');
4868                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4869                 }
4870             } elsif ($opt =~ m/^usb\d+/) {
4871                 die "skip\n";
4872                 # since we cannot reliably hot unplug usb devices
4873                 # we are disabling it
4874                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4875                 vm_deviceunplug($vmid, $conf, $opt);
4876             } elsif ($opt eq 'vcpus') {
4877                 die "skip\n" if !$hotplug_features->{cpu};
4878                 qemu_cpu_hotplug($vmid, $conf, undef);
4879             } elsif ($opt eq 'balloon') {
4880                 # enable balloon device is not hotpluggable
4881                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4882                 # here we reset the ballooning value to memory
4883                 my $balloon = $conf->{memory} || $defaults->{memory};
4884                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4885             } elsif ($fast_plug_option->{$opt}) {
4886                 # do nothing
4887             } elsif ($opt =~ m/^net(\d+)$/) {
4888                 die "skip\n" if !$hotplug_features->{network};
4889                 vm_deviceunplug($vmid, $conf, $opt);
4890             } elsif (is_valid_drivename($opt)) {
4891                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4892                 vm_deviceunplug($vmid, $conf, $opt);
4893                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4894             } elsif ($opt =~ m/^memory$/) {
4895                 die "skip\n" if !$hotplug_features->{memory};
4896                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4897             } elsif ($opt eq 'cpuunits') {
4898                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4899             } elsif ($opt eq 'cpulimit') {
4900                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4901             } else {
4902                 die "skip\n";
4903             }
4904         };
4905         if (my $err = $@) {
4906             &$add_error($opt, $err) if $err ne "skip\n";
4907         } else {
4908             # save new config if hotplug was successful
4909             delete $conf->{$opt};
4910             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4911             PVE::QemuConfig->write_config($vmid, $conf);
4912             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4913         }
4914     }
4915
4916     my $apply_pending_cloudinit;
4917     $apply_pending_cloudinit = sub {
4918         my ($key, $value) = @_;
4919         $apply_pending_cloudinit = sub {}; # once is enough
4920
4921         my @cloudinit_opts = keys %$confdesc_cloudinit;
4922         foreach my $opt (keys %{$conf->{pending}}) {
4923             next if !grep { $_ eq $opt } @cloudinit_opts;
4924             $conf->{$opt} = delete $conf->{pending}->{$opt};
4925         }
4926
4927         my $new_conf = { %$conf };
4928         $new_conf->{$key} = $value;
4929         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4930     };
4931
4932     foreach my $opt (keys %{$conf->{pending}}) {
4933         next if $selection && !$selection->{$opt};
4934         my $value = $conf->{pending}->{$opt};
4935         eval {
4936             if ($opt eq 'hotplug') {
4937                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4938             } elsif ($opt eq 'tablet') {
4939                 die "skip\n" if !$hotplug_features->{usb};
4940                 if ($value == 1) {
4941                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4942                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4943                         if $arch eq 'aarch64';
4944                 } elsif ($value == 0) {
4945                     vm_deviceunplug($vmid, $conf, 'tablet');
4946                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4947                 }
4948             } elsif ($opt =~ m/^usb\d+$/) {
4949                 die "skip\n";
4950                 # since we cannot reliably hot unplug usb devices
4951                 # we are disabling it
4952                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4953                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4954                 die "skip\n" if !$d;
4955                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4956             } elsif ($opt eq 'vcpus') {
4957                 die "skip\n" if !$hotplug_features->{cpu};
4958                 qemu_cpu_hotplug($vmid, $conf, $value);
4959             } elsif ($opt eq 'balloon') {
4960                 # enable/disable balloning device is not hotpluggable
4961                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4962                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4963                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4964
4965                 # allow manual ballooning if shares is set to zero
4966                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4967                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4968                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4969                 }
4970             } elsif ($opt =~ m/^net(\d+)$/) {
4971                 # some changes can be done without hotplug
4972                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4973                                     $vmid, $opt, $value, $arch, $machine_type);
4974             } elsif (is_valid_drivename($opt)) {
4975                 # some changes can be done without hotplug
4976                 my $drive = parse_drive($opt, $value);
4977                 if (drive_is_cloudinit($drive)) {
4978                     &$apply_pending_cloudinit($opt, $value);
4979                 }
4980                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4981                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4982             } elsif ($opt =~ m/^memory$/) { #dimms
4983                 die "skip\n" if !$hotplug_features->{memory};
4984                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4985             } elsif ($opt eq 'cpuunits') {
4986                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4987             } elsif ($opt eq 'cpulimit') {
4988                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4989                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4990             } else {
4991                 die "skip\n";  # skip non-hot-pluggable options
4992             }
4993         };
4994         if (my $err = $@) {
4995             &$add_error($opt, $err) if $err ne "skip\n";
4996         } else {
4997             # save new config if hotplug was successful
4998             $conf->{$opt} = $value;
4999             delete $conf->{pending}->{$opt};
5000             PVE::QemuConfig->write_config($vmid, $conf);
5001             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5002         }
5003     }
5004 }
5005
5006 sub try_deallocate_drive {
5007     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5008
5009     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5010         my $volid = $drive->{file};
5011         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5012             my $sid = PVE::Storage::parse_volume_id($volid);
5013             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5014
5015             # check if the disk is really unused
5016             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5017                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5018             PVE::Storage::vdisk_free($storecfg, $volid);
5019             return 1;
5020         } else {
5021             # If vm is not owner of this disk remove from config
5022             return 1;
5023         }
5024     }
5025
5026     return undef;
5027 }
5028
5029 sub vmconfig_delete_or_detach_drive {
5030     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5031
5032     my $drive = parse_drive($opt, $conf->{$opt});
5033
5034     my $rpcenv = PVE::RPCEnvironment::get();
5035     my $authuser = $rpcenv->get_user();
5036
5037     if ($force) {
5038         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5039         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5040     } else {
5041         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5042     }
5043 }
5044
5045
5046
5047 sub vmconfig_apply_pending {
5048     my ($vmid, $conf, $storecfg) = @_;
5049
5050     # cold plug
5051
5052     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5053     foreach my $opt (sort keys %$pending_delete_hash) {
5054         die "internal error" if $opt =~ m/^unused/;
5055         my $force = $pending_delete_hash->{$opt}->{force};
5056         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5057         if (!defined($conf->{$opt})) {
5058             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5059             PVE::QemuConfig->write_config($vmid, $conf);
5060         } elsif (is_valid_drivename($opt)) {
5061             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5062             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5063             delete $conf->{$opt};
5064             PVE::QemuConfig->write_config($vmid, $conf);
5065         } else {
5066             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5067             delete $conf->{$opt};
5068             PVE::QemuConfig->write_config($vmid, $conf);
5069         }
5070     }
5071
5072     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5073
5074     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5075         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5076
5077         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5078             # skip if nothing changed
5079         } elsif (is_valid_drivename($opt)) {
5080             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5081                 if defined($conf->{$opt});
5082             $conf->{$opt} = $conf->{pending}->{$opt};
5083         } else {
5084             $conf->{$opt} = $conf->{pending}->{$opt};
5085         }
5086
5087         delete $conf->{pending}->{$opt};
5088         PVE::QemuConfig->write_config($vmid, $conf);
5089     }
5090 }
5091
5092 my $safe_num_ne = sub {
5093     my ($a, $b) = @_;
5094
5095     return 0 if !defined($a) && !defined($b);
5096     return 1 if !defined($a);
5097     return 1 if !defined($b);
5098
5099     return $a != $b;
5100 };
5101
5102 my $safe_string_ne = sub {
5103     my ($a, $b) = @_;
5104
5105     return 0 if !defined($a) && !defined($b);
5106     return 1 if !defined($a);
5107     return 1 if !defined($b);
5108
5109     return $a ne $b;
5110 };
5111
5112 sub vmconfig_update_net {
5113     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5114
5115     my $newnet = parse_net($value);
5116
5117     if ($conf->{$opt}) {
5118         my $oldnet = parse_net($conf->{$opt});
5119
5120         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5121             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5122             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5123             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5124
5125             # for non online change, we try to hot-unplug
5126             die "skip\n" if !$hotplug;
5127             vm_deviceunplug($vmid, $conf, $opt);
5128         } else {
5129
5130             die "internal error" if $opt !~ m/net(\d+)/;
5131             my $iface = "tap${vmid}i$1";
5132
5133             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5134                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5135                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5136                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5137                 PVE::Network::tap_unplug($iface);
5138                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5139             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5140                 # Rate can be applied on its own but any change above needs to
5141                 # include the rate in tap_plug since OVS resets everything.
5142                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5143             }
5144
5145             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5146                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5147             }
5148
5149             return 1;
5150         }
5151     }
5152
5153     if ($hotplug) {
5154         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5155     } else {
5156         die "skip\n";
5157     }
5158 }
5159
5160 sub vmconfig_update_disk {
5161     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5162
5163     # fixme: do we need force?
5164
5165     my $drive = parse_drive($opt, $value);
5166
5167     if ($conf->{$opt}) {
5168
5169         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5170
5171             my $media = $drive->{media} || 'disk';
5172             my $oldmedia = $old_drive->{media} || 'disk';
5173             die "unable to change media type\n" if $media ne $oldmedia;
5174
5175             if (!drive_is_cdrom($old_drive)) {
5176
5177                 if ($drive->{file} ne $old_drive->{file}) {
5178
5179                     die "skip\n" if !$hotplug;
5180
5181                     # unplug and register as unused
5182                     vm_deviceunplug($vmid, $conf, $opt);
5183                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5184
5185                 } else {
5186                     # update existing disk
5187
5188                     # skip non hotpluggable value
5189                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5190                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5191                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5192                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5193                         die "skip\n";
5194                     }
5195
5196                     # apply throttle
5197                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5198                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5199                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5200                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5201                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5202                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5203                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5204                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5205                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5206                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5207                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5208                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5209                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5210                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5211                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5212                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5213                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5214                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5215
5216                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5217                                                    ($drive->{mbps} || 0)*1024*1024,
5218                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5219                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5220                                                    $drive->{iops} || 0,
5221                                                    $drive->{iops_rd} || 0,
5222                                                    $drive->{iops_wr} || 0,
5223                                                    ($drive->{mbps_max} || 0)*1024*1024,
5224                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5225                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5226                                                    $drive->{iops_max} || 0,
5227                                                    $drive->{iops_rd_max} || 0,
5228                                                    $drive->{iops_wr_max} || 0,
5229                                                    $drive->{bps_max_length} || 1,
5230                                                    $drive->{bps_rd_max_length} || 1,
5231                                                    $drive->{bps_wr_max_length} || 1,
5232                                                    $drive->{iops_max_length} || 1,
5233                                                    $drive->{iops_rd_max_length} || 1,
5234                                                    $drive->{iops_wr_max_length} || 1);
5235
5236                     }
5237
5238                     return 1;
5239                 }
5240
5241             } else { # cdrom
5242
5243                 if ($drive->{file} eq 'none') {
5244                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5245                     if (drive_is_cloudinit($old_drive)) {
5246                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5247                     }
5248                 } else {
5249                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5250                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5251                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5252                 }
5253
5254                 return 1;
5255             }
5256         }
5257     }
5258
5259     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5260     # hotplug new disks
5261     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5262     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5263 }
5264
5265 sub vm_start {
5266     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5267         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5268
5269     PVE::QemuConfig->lock_config($vmid, sub {
5270         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5271
5272         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5273
5274         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5275
5276         PVE::QemuConfig->check_lock($conf)
5277             if !($skiplock || $is_suspended);
5278
5279         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5280
5281         # clean up leftover reboot request files
5282         eval { clear_reboot_request($vmid); };
5283         warn $@ if $@;
5284
5285         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5286             vmconfig_apply_pending($vmid, $conf, $storecfg);
5287             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5288         }
5289
5290         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5291
5292         my $defaults = load_defaults();
5293
5294         # set environment variable useful inside network script
5295         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5296
5297         my $local_volumes = {};
5298
5299         if ($targetstorage) {
5300             foreach_drive($conf, sub {
5301                 my ($ds, $drive) = @_;
5302
5303                 return if drive_is_cdrom($drive);
5304
5305                 my $volid = $drive->{file};
5306
5307                 return if !$volid;
5308
5309                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5310
5311                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5312                 return if $scfg->{shared};
5313                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5314             });
5315
5316             my $format = undef;
5317
5318             foreach my $opt (sort keys %$local_volumes) {
5319
5320                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5321                 my $drive = parse_drive($opt, $conf->{$opt});
5322
5323                 #if remote storage is specified, use default format
5324                 if ($targetstorage && $targetstorage ne "1") {
5325                     $storeid = $targetstorage;
5326                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5327                     $format = $defFormat;
5328                 } else {
5329                     #else we use same format than original
5330                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5331                     $format = qemu_img_format($scfg, $volid);
5332                 }
5333
5334                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5335                 my $newdrive = $drive;
5336                 $newdrive->{format} = $format;
5337                 $newdrive->{file} = $newvolid;
5338                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5339                 $local_volumes->{$opt} = $drivestr;
5340                 #pass drive to conf for command line
5341                 $conf->{$opt} = $drivestr;
5342             }
5343         }
5344
5345         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5346
5347         if ($is_suspended) {
5348             # enforce machine type on suspended vm to ensure HW compatibility
5349             $forcemachine = $conf->{runningmachine};
5350             print "Resuming suspended VM\n";
5351         }
5352
5353         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5354
5355         my $migrate_uri;
5356         if ($statefile) {
5357             if ($statefile eq 'tcp') {
5358                 my $localip = "localhost";
5359                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5360                 my $nodename = PVE::INotify::nodename();
5361
5362                 if (!defined($migration_type)) {
5363                     if (defined($datacenterconf->{migration}->{type})) {
5364                         $migration_type = $datacenterconf->{migration}->{type};
5365                     } else {
5366                         $migration_type = 'secure';
5367                     }
5368                 }
5369
5370                 if ($migration_type eq 'insecure') {
5371                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5372                     if ($migrate_network_addr) {
5373                         $localip = $migrate_network_addr;
5374                     } else {
5375                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5376                     }
5377
5378                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5379                 }
5380
5381                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5382                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5383                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5384                 push @$cmd, '-incoming', $migrate_uri;
5385                 push @$cmd, '-S';
5386
5387             } elsif ($statefile eq 'unix') {
5388                 # should be default for secure migrations as a ssh TCP forward
5389                 # tunnel is not deterministic reliable ready and fails regurarly
5390                 # to set up in time, so use UNIX socket forwards
5391                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5392                 unlink $socket_addr;
5393
5394                 $migrate_uri = "unix:$socket_addr";
5395
5396                 push @$cmd, '-incoming', $migrate_uri;
5397                 push @$cmd, '-S';
5398
5399             } elsif (-e $statefile) {
5400                 push @$cmd, '-loadstate', $statefile;
5401             } else {
5402                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5403                 push @$vollist, $statefile;
5404                 push @$cmd, '-loadstate', $statepath;
5405             }
5406         } elsif ($paused) {
5407             push @$cmd, '-S';
5408         }
5409
5410         # host pci devices
5411         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5412           my $d = parse_hostpci($conf->{"hostpci$i"});
5413           next if !$d;
5414           my $pcidevices = $d->{pciid};
5415           foreach my $pcidevice (@$pcidevices) {
5416                 my $pciid = $pcidevice->{id};
5417
5418                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5419                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5420                 die "no pci device info for device '$pciid'\n" if !$info;
5421
5422                 if ($d->{mdev}) {
5423                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5424                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5425                 } else {
5426                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5427                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5428                     die "can't reset pci device '$pciid'\n"
5429                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5430                 }
5431           }
5432         }
5433
5434         PVE::Storage::activate_volumes($storecfg, $vollist);
5435
5436         eval {
5437             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5438                 outfunc => sub {}, errfunc => sub {});
5439         };
5440         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5441         # timeout should be more than enough here...
5442         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5443
5444         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5445                                                   : $defaults->{cpuunits};
5446
5447         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5448         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5449
5450         my %properties = (
5451             Slice => 'qemu.slice',
5452             KillMode => 'none',
5453             CPUShares => $cpuunits
5454         );
5455
5456         if (my $cpulimit = $conf->{cpulimit}) {
5457             $properties{CPUQuota} = int($cpulimit * 100);
5458         }
5459         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5460
5461         my $run_qemu = sub {
5462             PVE::Tools::run_fork sub {
5463                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5464                 run_command($cmd, %run_params);
5465             };
5466         };
5467
5468         if ($conf->{hugepages}) {
5469
5470             my $code = sub {
5471                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5472                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5473
5474                 PVE::QemuServer::Memory::hugepages_mount();
5475                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5476
5477                 eval { $run_qemu->() };
5478                 if (my $err = $@) {
5479                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5480                     die $err;
5481                 }
5482
5483                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5484             };
5485             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5486
5487         } else {
5488             eval { $run_qemu->() };
5489         }
5490
5491         if (my $err = $@) {
5492             # deactivate volumes if start fails
5493             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5494             die "start failed: $err";
5495         }
5496
5497         print "migration listens on $migrate_uri\n" if $migrate_uri;
5498
5499         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5500             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5501             warn $@ if $@;
5502         }
5503
5504         #start nbd server for storage migration
5505         if ($targetstorage) {
5506             my $nodename = PVE::INotify::nodename();
5507             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5508             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5509             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5510             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5511
5512             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5513
5514             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5515
5516             foreach my $opt (sort keys %$local_volumes) {
5517                 my $volid = $local_volumes->{$opt};
5518                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5519                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5520                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5521             }
5522         }
5523
5524         if ($migratedfrom) {
5525             eval {
5526                 set_migration_caps($vmid);
5527             };
5528             warn $@ if $@;
5529
5530             if ($spice_port) {
5531                 print "spice listens on port $spice_port\n";
5532                 if ($spice_ticket) {
5533                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5534                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5535                 }
5536             }
5537
5538         } else {
5539             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5540                 if !$statefile && $conf->{balloon};
5541
5542             foreach my $opt (keys %$conf) {
5543                 next if $opt !~  m/^net\d+$/;
5544                 my $nicconf = parse_net($conf->{$opt});
5545                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5546             }
5547         }
5548
5549         vm_mon_cmd_nocheck($vmid, 'qom-set',
5550                     path => "machine/peripheral/balloon0",
5551                     property => "guest-stats-polling-interval",
5552                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5553
5554         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5555             print "Resumed VM, removing state\n";
5556             delete $conf->@{qw(lock vmstate runningmachine)};
5557             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5558             PVE::Storage::vdisk_free($storecfg, $vmstate);
5559             PVE::QemuConfig->write_config($vmid, $conf);
5560         }
5561
5562         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5563     });
5564 }
5565
5566 sub vm_mon_cmd {
5567     my ($vmid, $execute, %params) = @_;
5568
5569     my $cmd = { execute => $execute, arguments => \%params };
5570     vm_qmp_command($vmid, $cmd);
5571 }
5572
5573 sub vm_mon_cmd_nocheck {
5574     my ($vmid, $execute, %params) = @_;
5575
5576     my $cmd = { execute => $execute, arguments => \%params };
5577     vm_qmp_command($vmid, $cmd, 1);
5578 }
5579
5580 sub vm_qmp_command {
5581     my ($vmid, $cmd, $nocheck) = @_;
5582
5583     my $res;
5584
5585     my $timeout;
5586     if ($cmd->{arguments}) {
5587         $timeout = delete $cmd->{arguments}->{timeout};
5588     }
5589
5590     eval {
5591         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5592         my $sname = qmp_socket($vmid);
5593         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5594             my $qmpclient = PVE::QMPClient->new();
5595
5596             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5597         } else {
5598             die "unable to open monitor socket\n";
5599         }
5600     };
5601     if (my $err = $@) {
5602         syslog("err", "VM $vmid qmp command failed - $err");
5603         die $err;
5604     }
5605
5606     return $res;
5607 }
5608
5609 sub vm_human_monitor_command {
5610     my ($vmid, $cmdline) = @_;
5611
5612     my $cmd = {
5613         execute => 'human-monitor-command',
5614         arguments => { 'command-line' => $cmdline},
5615     };
5616
5617     return vm_qmp_command($vmid, $cmd);
5618 }
5619
5620 sub vm_commandline {
5621     my ($storecfg, $vmid, $snapname) = @_;
5622
5623     my $conf = PVE::QemuConfig->load_config($vmid);
5624
5625     if ($snapname) {
5626         my $snapshot = $conf->{snapshots}->{$snapname};
5627         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5628
5629         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5630
5631         $conf = $snapshot;
5632     }
5633
5634     my $defaults = load_defaults();
5635
5636     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5637
5638     return PVE::Tools::cmd2string($cmd);
5639 }
5640
5641 sub vm_reset {
5642     my ($vmid, $skiplock) = @_;
5643
5644     PVE::QemuConfig->lock_config($vmid, sub {
5645
5646         my $conf = PVE::QemuConfig->load_config($vmid);
5647
5648         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5649
5650         vm_mon_cmd($vmid, "system_reset");
5651     });
5652 }
5653
5654 sub get_vm_volumes {
5655     my ($conf) = @_;
5656
5657     my $vollist = [];
5658     foreach_volid($conf, sub {
5659         my ($volid, $attr) = @_;
5660
5661         return if $volid =~ m|^/|;
5662
5663         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5664         return if !$sid;
5665
5666         push @$vollist, $volid;
5667     });
5668
5669     return $vollist;
5670 }
5671
5672 sub vm_stop_cleanup {
5673     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5674
5675     eval {
5676
5677         if (!$keepActive) {
5678             my $vollist = get_vm_volumes($conf);
5679             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5680         }
5681
5682         foreach my $ext (qw(mon qmp pid vnc qga)) {
5683             unlink "/var/run/qemu-server/${vmid}.$ext";
5684         }
5685
5686         if ($conf->{ivshmem}) {
5687             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5688             # just delete it for now, VMs which have this already open do not
5689             # are affected, but new VMs will get a separated one. If this
5690             # becomes an issue we either add some sort of ref-counting or just
5691             # add a "don't delete on stop" flag to the ivshmem format.
5692             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5693         }
5694
5695         foreach my $key (keys %$conf) {
5696             next if $key !~ m/^hostpci(\d+)$/;
5697             my $hostpciindex = $1;
5698             my $d = parse_hostpci($conf->{$key});
5699             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5700
5701             foreach my $pci (@{$d->{pciid}}) {
5702                 my $pciid = $pci->{id};
5703                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5704             }
5705         }
5706
5707         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5708     };
5709     warn $@ if $@; # avoid errors - just warn
5710 }
5711
5712 # call only in locked context
5713 sub _do_vm_stop {
5714     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5715
5716     my $pid = check_running($vmid, $nocheck);
5717     return if !$pid;
5718
5719     my $conf;
5720     if (!$nocheck) {
5721         $conf = PVE::QemuConfig->load_config($vmid);
5722         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5723         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5724             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5725             $timeout = $opts->{down} if $opts->{down};
5726         }
5727         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5728     }
5729
5730     eval {
5731         if ($shutdown) {
5732             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5733                 vm_qmp_command($vmid, {
5734                         execute => "guest-shutdown",
5735                         arguments => { timeout => $timeout }
5736                     }, $nocheck);
5737             } else {
5738                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5739             }
5740         } else {
5741             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5742         }
5743     };
5744     my $err = $@;
5745
5746     if (!$err) {
5747         $timeout = 60 if !defined($timeout);
5748
5749         my $count = 0;
5750         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5751             $count++;
5752             sleep 1;
5753         }
5754
5755         if ($count >= $timeout) {
5756             if ($force) {
5757                 warn "VM still running - terminating now with SIGTERM\n";
5758                 kill 15, $pid;
5759             } else {
5760                 die "VM quit/powerdown failed - got timeout\n";
5761             }
5762         } else {
5763             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5764             return;
5765         }
5766     } else {
5767         if ($force) {
5768             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5769             kill 15, $pid;
5770         } else {
5771             die "VM quit/powerdown failed\n";
5772         }
5773     }
5774
5775     # wait again
5776     $timeout = 10;
5777
5778     my $count = 0;
5779     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5780         $count++;
5781         sleep 1;
5782     }
5783
5784     if ($count >= $timeout) {
5785         warn "VM still running - terminating now with SIGKILL\n";
5786         kill 9, $pid;
5787         sleep 1;
5788     }
5789
5790     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5791 }
5792
5793 # Note: use $nocheck to skip tests if VM configuration file exists.
5794 # We need that when migration VMs to other nodes (files already moved)
5795 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5796 sub vm_stop {
5797     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5798
5799     $force = 1 if !defined($force) && !$shutdown;
5800
5801     if ($migratedfrom){
5802         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5803         kill 15, $pid if $pid;
5804         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5805         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5806         return;
5807     }
5808
5809     PVE::QemuConfig->lock_config($vmid, sub {
5810         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5811    });
5812 }
5813
5814 sub vm_reboot {
5815     my ($vmid, $timeout) = @_;
5816
5817     PVE::QemuConfig->lock_config($vmid, sub {
5818
5819         # only reboot if running, as qmeventd starts it again on a stop event
5820         return if !check_running($vmid);
5821
5822         create_reboot_request($vmid);
5823
5824         my $storecfg = PVE::Storage::config();
5825         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5826
5827    });
5828 }
5829
5830 sub vm_suspend {
5831     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5832
5833     my $conf;
5834     my $path;
5835     my $storecfg;
5836     my $vmstate;
5837
5838     PVE::QemuConfig->lock_config($vmid, sub {
5839
5840         $conf = PVE::QemuConfig->load_config($vmid);
5841
5842         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5843         PVE::QemuConfig->check_lock($conf)
5844             if !($skiplock || $is_backing_up);
5845
5846         die "cannot suspend to disk during backup\n"
5847             if $is_backing_up && $includestate;
5848
5849         if ($includestate) {
5850             $conf->{lock} = 'suspending';
5851             my $date = strftime("%Y-%m-%d", localtime(time()));
5852             $storecfg = PVE::Storage::config();
5853             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5854             $path = PVE::Storage::path($storecfg, $vmstate);
5855             PVE::QemuConfig->write_config($vmid, $conf);
5856         } else {
5857             vm_mon_cmd($vmid, "stop");
5858         }
5859     });
5860
5861     if ($includestate) {
5862         # save vm state
5863         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5864
5865         eval {
5866             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5867             for(;;) {
5868                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5869                 if (!$state->{status}) {
5870                     die "savevm not active\n";
5871                 } elsif ($state->{status} eq 'active') {
5872                     sleep(1);
5873                     next;
5874                 } elsif ($state->{status} eq 'completed') {
5875                     print "State saved, quitting\n";
5876                     last;
5877                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5878                     die "query-savevm failed with error '$state->{error}'\n"
5879                 } else {
5880                     die "query-savevm returned status '$state->{status}'\n";
5881                 }
5882             }
5883         };
5884         my $err = $@;
5885
5886         PVE::QemuConfig->lock_config($vmid, sub {
5887             $conf = PVE::QemuConfig->load_config($vmid);
5888             if ($err) {
5889                 # cleanup, but leave suspending lock, to indicate something went wrong
5890                 eval {
5891                     vm_mon_cmd($vmid, "savevm-end");
5892                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5893                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5894                     delete $conf->@{qw(vmstate runningmachine)};
5895                     PVE::QemuConfig->write_config($vmid, $conf);
5896                 };
5897                 warn $@ if $@;
5898                 die $err;
5899             }
5900
5901             die "lock changed unexpectedly\n"
5902                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5903
5904             vm_qmp_command($vmid, { execute => "quit" });
5905             $conf->{lock} = 'suspended';
5906             PVE::QemuConfig->write_config($vmid, $conf);
5907         });
5908     }
5909 }
5910
5911 sub vm_resume {
5912     my ($vmid, $skiplock, $nocheck) = @_;
5913
5914     PVE::QemuConfig->lock_config($vmid, sub {
5915         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5916         my $res = $vm_mon_cmd->($vmid, 'query-status');
5917         my $resume_cmd = 'cont';
5918
5919         if ($res->{status} && $res->{status} eq 'suspended') {
5920             $resume_cmd = 'system_wakeup';
5921         }
5922
5923         if (!$nocheck) {
5924
5925             my $conf = PVE::QemuConfig->load_config($vmid);
5926
5927             PVE::QemuConfig->check_lock($conf)
5928                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5929         }
5930
5931         $vm_mon_cmd->($vmid, $resume_cmd);
5932     });
5933 }
5934
5935 sub vm_sendkey {
5936     my ($vmid, $skiplock, $key) = @_;
5937
5938     PVE::QemuConfig->lock_config($vmid, sub {
5939
5940         my $conf = PVE::QemuConfig->load_config($vmid);
5941
5942         # there is no qmp command, so we use the human monitor command
5943         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5944         die $res if $res ne '';
5945     });
5946 }
5947
5948 sub vm_destroy {
5949     my ($storecfg, $vmid, $skiplock) = @_;
5950
5951     PVE::QemuConfig->lock_config($vmid, sub {
5952
5953         my $conf = PVE::QemuConfig->load_config($vmid);
5954
5955         if (!check_running($vmid)) {
5956             destroy_vm($storecfg, $vmid, undef, $skiplock);
5957         } else {
5958             die "VM $vmid is running - destroy failed\n";
5959         }
5960     });
5961 }
5962
5963 # vzdump restore implementaion
5964
5965 sub tar_archive_read_firstfile {
5966     my $archive = shift;
5967
5968     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5969
5970     # try to detect archive type first
5971     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5972         die "unable to open file '$archive'\n";
5973     my $firstfile = <$fh>;
5974     kill 15, $pid;
5975     close $fh;
5976
5977     die "ERROR: archive contaions no data\n" if !$firstfile;
5978     chomp $firstfile;
5979
5980     return $firstfile;
5981 }
5982
5983 sub tar_restore_cleanup {
5984     my ($storecfg, $statfile) = @_;
5985
5986     print STDERR "starting cleanup\n";
5987
5988     if (my $fd = IO::File->new($statfile, "r")) {
5989         while (defined(my $line = <$fd>)) {
5990             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5991                 my $volid = $2;
5992                 eval {
5993                     if ($volid =~ m|^/|) {
5994                         unlink $volid || die 'unlink failed\n';
5995                     } else {
5996                         PVE::Storage::vdisk_free($storecfg, $volid);
5997                     }
5998                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5999                 };
6000                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6001             } else {
6002                 print STDERR "unable to parse line in statfile - $line";
6003             }
6004         }
6005         $fd->close();
6006     }
6007 }
6008
6009 sub restore_archive {
6010     my ($archive, $vmid, $user, $opts) = @_;
6011
6012     my $format = $opts->{format};
6013     my $comp;
6014
6015     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6016         $format = 'tar' if !$format;
6017         $comp = 'gzip';
6018     } elsif ($archive =~ m/\.tar$/) {
6019         $format = 'tar' if !$format;
6020     } elsif ($archive =~ m/.tar.lzo$/) {
6021         $format = 'tar' if !$format;
6022         $comp = 'lzop';
6023     } elsif ($archive =~ m/\.vma$/) {
6024         $format = 'vma' if !$format;
6025     } elsif ($archive =~ m/\.vma\.gz$/) {
6026         $format = 'vma' if !$format;
6027         $comp = 'gzip';
6028     } elsif ($archive =~ m/\.vma\.lzo$/) {
6029         $format = 'vma' if !$format;
6030         $comp = 'lzop';
6031     } else {
6032         $format = 'vma' if !$format; # default
6033     }
6034
6035     # try to detect archive format
6036     if ($format eq 'tar') {
6037         return restore_tar_archive($archive, $vmid, $user, $opts);
6038     } else {
6039         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6040     }
6041 }
6042
6043 sub restore_update_config_line {
6044     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6045
6046     return if $line =~ m/^\#qmdump\#/;
6047     return if $line =~ m/^\#vzdump\#/;
6048     return if $line =~ m/^lock:/;
6049     return if $line =~ m/^unused\d+:/;
6050     return if $line =~ m/^parent:/;
6051
6052     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6053     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6054         # try to convert old 1.X settings
6055         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6056         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6057             my ($model, $macaddr) = split(/\=/, $devconfig);
6058             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6059             my $net = {
6060                 model => $model,
6061                 bridge => "vmbr$ind",
6062                 macaddr => $macaddr,
6063             };
6064             my $netstr = print_net($net);
6065
6066             print $outfd "net$cookie->{netcount}: $netstr\n";
6067             $cookie->{netcount}++;
6068         }
6069     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6070         my ($id, $netstr) = ($1, $2);
6071         my $net = parse_net($netstr);
6072         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6073         $netstr = print_net($net);
6074         print $outfd "$id: $netstr\n";
6075     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6076         my $virtdev = $1;
6077         my $value = $3;
6078         my $di = parse_drive($virtdev, $value);
6079         if (defined($di->{backup}) && !$di->{backup}) {
6080             print $outfd "#$line";
6081         } elsif ($map->{$virtdev}) {
6082             delete $di->{format}; # format can change on restore
6083             $di->{file} = $map->{$virtdev};
6084             $value = print_drive($vmid, $di);
6085             print $outfd "$virtdev: $value\n";
6086         } else {
6087             print $outfd $line;
6088         }
6089     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6090         my $vmgenid = $1;
6091         if ($vmgenid ne '0') {
6092             # always generate a new vmgenid if there was a valid one setup
6093             $vmgenid = generate_uuid();
6094         }
6095         print $outfd "vmgenid: $vmgenid\n";
6096     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6097         my ($uuid, $uuid_str);
6098         UUID::generate($uuid);
6099         UUID::unparse($uuid, $uuid_str);
6100         my $smbios1 = parse_smbios1($2);
6101         $smbios1->{uuid} = $uuid_str;
6102         print $outfd $1.print_smbios1($smbios1)."\n";
6103     } else {
6104         print $outfd $line;
6105     }
6106 }
6107
6108 sub scan_volids {
6109     my ($cfg, $vmid) = @_;
6110
6111     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6112
6113     my $volid_hash = {};
6114     foreach my $storeid (keys %$info) {
6115         foreach my $item (@{$info->{$storeid}}) {
6116             next if !($item->{volid} && $item->{size});
6117             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6118             $volid_hash->{$item->{volid}} = $item;
6119         }
6120     }
6121
6122     return $volid_hash;
6123 }
6124
6125 sub is_volume_in_use {
6126     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6127
6128     my $path = PVE::Storage::path($storecfg, $volid);
6129
6130     my $scan_config = sub {
6131         my ($cref, $snapname) = @_;
6132
6133         foreach my $key (keys %$cref) {
6134             my $value = $cref->{$key};
6135             if (is_valid_drivename($key)) {
6136                 next if $skip_drive && $key eq $skip_drive;
6137                 my $drive = parse_drive($key, $value);
6138                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6139                 return 1 if $volid eq $drive->{file};
6140                 if ($drive->{file} =~ m!^/!) {
6141                     return 1 if $drive->{file} eq $path;
6142                 } else {
6143                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6144                     next if !$storeid;
6145                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6146                     next if !$scfg;
6147                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6148                 }
6149             }
6150         }
6151
6152         return 0;
6153     };
6154
6155     return 1 if &$scan_config($conf);
6156
6157     undef $skip_drive;
6158
6159     foreach my $snapname (keys %{$conf->{snapshots}}) {
6160         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6161     }
6162
6163     return 0;
6164 }
6165
6166 sub update_disksize {
6167     my ($vmid, $conf, $volid_hash) = @_;
6168
6169     my $changes;
6170     my $prefix = "VM $vmid:";
6171
6172     # used and unused disks
6173     my $referenced = {};
6174
6175     # Note: it is allowed to define multiple storages with same path (alias), so
6176     # we need to check both 'volid' and real 'path' (two different volid can point
6177     # to the same path).
6178
6179     my $referencedpath = {};
6180
6181     # update size info
6182     foreach my $opt (keys %$conf) {
6183         if (is_valid_drivename($opt)) {
6184             my $drive = parse_drive($opt, $conf->{$opt});
6185             my $volid = $drive->{file};
6186             next if !$volid;
6187
6188             $referenced->{$volid} = 1;
6189             if ($volid_hash->{$volid} &&
6190                 (my $path = $volid_hash->{$volid}->{path})) {
6191                 $referencedpath->{$path} = 1;
6192             }
6193
6194             next if drive_is_cdrom($drive);
6195             next if !$volid_hash->{$volid};
6196
6197             $drive->{size} = $volid_hash->{$volid}->{size};
6198             my $new = print_drive($vmid, $drive);
6199             if ($new ne $conf->{$opt}) {
6200                 $changes = 1;
6201                 $conf->{$opt} = $new;
6202                 print "$prefix update disk '$opt' information.\n";
6203             }
6204         }
6205     }
6206
6207     # remove 'unusedX' entry if volume is used
6208     foreach my $opt (keys %$conf) {
6209         next if $opt !~ m/^unused\d+$/;
6210         my $volid = $conf->{$opt};
6211         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6212         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6213             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6214             $changes = 1;
6215             delete $conf->{$opt};
6216         }
6217
6218         $referenced->{$volid} = 1;
6219         $referencedpath->{$path} = 1 if $path;
6220     }
6221
6222     foreach my $volid (sort keys %$volid_hash) {
6223         next if $volid =~ m/vm-$vmid-state-/;
6224         next if $referenced->{$volid};
6225         my $path = $volid_hash->{$volid}->{path};
6226         next if !$path; # just to be sure
6227         next if $referencedpath->{$path};
6228         $changes = 1;
6229         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6230         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6231         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6232     }
6233
6234     return $changes;
6235 }
6236
6237 sub rescan {
6238     my ($vmid, $nolock, $dryrun) = @_;
6239
6240     my $cfg = PVE::Storage::config();
6241
6242     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6243     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6244     foreach my $stor (keys %{$cfg->{ids}}) {
6245         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6246     }
6247
6248     print "rescan volumes...\n";
6249     my $volid_hash = scan_volids($cfg, $vmid);
6250
6251     my $updatefn =  sub {
6252         my ($vmid) = @_;
6253
6254         my $conf = PVE::QemuConfig->load_config($vmid);
6255
6256         PVE::QemuConfig->check_lock($conf);
6257
6258         my $vm_volids = {};
6259         foreach my $volid (keys %$volid_hash) {
6260             my $info = $volid_hash->{$volid};
6261             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6262         }
6263
6264         my $changes = update_disksize($vmid, $conf, $vm_volids);
6265
6266         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6267     };
6268
6269     if (defined($vmid)) {
6270         if ($nolock) {
6271             &$updatefn($vmid);
6272         } else {
6273             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6274         }
6275     } else {
6276         my $vmlist = config_list();
6277         foreach my $vmid (keys %$vmlist) {
6278             if ($nolock) {
6279                 &$updatefn($vmid);
6280             } else {
6281                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6282             }
6283         }
6284     }
6285 }
6286
6287 sub restore_vma_archive {
6288     my ($archive, $vmid, $user, $opts, $comp) = @_;
6289
6290     my $readfrom = $archive;
6291
6292     my $cfg = PVE::Storage::config();
6293     my $commands = [];
6294     my $bwlimit = $opts->{bwlimit};
6295
6296     my $dbg_cmdstring = '';
6297     my $add_pipe = sub {
6298         my ($cmd) = @_;
6299         push @$commands, $cmd;
6300         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6301         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6302         $readfrom = '-';
6303     };
6304
6305     my $input = undef;
6306     if ($archive eq '-') {
6307         $input = '<&STDIN';
6308     } else {
6309         # If we use a backup from a PVE defined storage we also consider that
6310         # storage's rate limit:
6311         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6312         if (defined($volid)) {
6313             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6314             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6315             if ($readlimit) {
6316                 print STDERR "applying read rate limit: $readlimit\n";
6317                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6318                 $add_pipe->($cstream);
6319             }
6320         }
6321     }
6322
6323     if ($comp) {
6324         my $cmd;
6325         if ($comp eq 'gzip') {
6326             $cmd = ['zcat', $readfrom];
6327         } elsif ($comp eq 'lzop') {
6328             $cmd = ['lzop', '-d', '-c', $readfrom];
6329         } else {
6330             die "unknown compression method '$comp'\n";
6331         }
6332         $add_pipe->($cmd);
6333     }
6334
6335     my $tmpdir = "/var/tmp/vzdumptmp$$";
6336     rmtree $tmpdir;
6337
6338     # disable interrupts (always do cleanups)
6339     local $SIG{INT} =
6340         local $SIG{TERM} =
6341         local $SIG{QUIT} =
6342         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6343
6344     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6345     POSIX::mkfifo($mapfifo, 0600);
6346     my $fifofh;
6347
6348     my $openfifo = sub {
6349         open($fifofh, '>', $mapfifo) || die $!;
6350     };
6351
6352     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6353
6354     my $oldtimeout;
6355     my $timeout = 5;
6356
6357     my $devinfo = {};
6358
6359     my $rpcenv = PVE::RPCEnvironment::get();
6360
6361     my $conffile = PVE::QemuConfig->config_file($vmid);
6362     my $tmpfn = "$conffile.$$.tmp";
6363
6364     # Note: $oldconf is undef if VM does not exists
6365     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6366     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6367
6368     my %storage_limits;
6369
6370     my $print_devmap = sub {
6371         my $virtdev_hash = {};
6372
6373         my $cfgfn = "$tmpdir/qemu-server.conf";
6374
6375         # we can read the config - that is already extracted
6376         my $fh = IO::File->new($cfgfn, "r") ||
6377             "unable to read qemu-server.conf - $!\n";
6378
6379         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6380         if (-f $fwcfgfn) {
6381             my $pve_firewall_dir = '/etc/pve/firewall';
6382             mkdir $pve_firewall_dir; # make sure the dir exists
6383             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6384         }
6385
6386         while (defined(my $line = <$fh>)) {
6387             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6388                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6389                 die "archive does not contain data for drive '$virtdev'\n"
6390                     if !$devinfo->{$devname};
6391                 if (defined($opts->{storage})) {
6392                     $storeid = $opts->{storage} || 'local';
6393                 } elsif (!$storeid) {
6394                     $storeid = 'local';
6395                 }
6396                 $format = 'raw' if !$format;
6397                 $devinfo->{$devname}->{devname} = $devname;
6398                 $devinfo->{$devname}->{virtdev} = $virtdev;
6399                 $devinfo->{$devname}->{format} = $format;
6400                 $devinfo->{$devname}->{storeid} = $storeid;
6401
6402                 # check permission on storage
6403                 my $pool = $opts->{pool}; # todo: do we need that?
6404                 if ($user ne 'root@pam') {
6405                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6406                 }
6407
6408                 $storage_limits{$storeid} = $bwlimit;
6409
6410                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6411             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6412                 my $virtdev = $1;
6413                 my $drive = parse_drive($virtdev, $2);
6414                 if (drive_is_cloudinit($drive)) {
6415                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6416                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6417                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6418
6419                     my $d = {
6420                         format => $format,
6421                         storeid => $opts->{storage} // $storeid,
6422                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6423                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6424                         name => "vm-$vmid-cloudinit",
6425                         is_cloudinit => 1,
6426                     };
6427                     $virtdev_hash->{$virtdev} = $d;
6428                 }
6429             }
6430         }
6431
6432         foreach my $key (keys %storage_limits) {
6433             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6434             next if !$limit;
6435             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6436             $storage_limits{$key} = $limit * 1024;
6437         }
6438
6439         foreach my $devname (keys %$devinfo) {
6440             die "found no device mapping information for device '$devname'\n"
6441                 if !$devinfo->{$devname}->{virtdev};
6442         }
6443
6444         # create empty/temp config
6445         if ($oldconf) {
6446             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6447             foreach_drive($oldconf, sub {
6448                 my ($ds, $drive) = @_;
6449
6450                 return if drive_is_cdrom($drive, 1);
6451
6452                 my $volid = $drive->{file};
6453                 return if !$volid || $volid =~ m|^/|;
6454
6455                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6456                 return if !$path || !$owner || ($owner != $vmid);
6457
6458                 # Note: only delete disk we want to restore
6459                 # other volumes will become unused
6460                 if ($virtdev_hash->{$ds}) {
6461                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6462                     if (my $err = $@) {
6463                         warn $err;
6464                     }
6465                 }
6466             });
6467
6468             # delete vmstate files, after the restore we have no snapshots anymore
6469             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6470                 my $snap = $oldconf->{snapshots}->{$snapname};
6471                 if ($snap->{vmstate}) {
6472                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6473                     if (my $err = $@) {
6474                         warn $err;
6475                     }
6476                 }
6477             }
6478         }
6479
6480         my $map = {};
6481         foreach my $virtdev (sort keys %$virtdev_hash) {
6482             my $d = $virtdev_hash->{$virtdev};
6483             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6484             my $storeid = $d->{storeid};
6485             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6486
6487             my $map_opts = '';
6488             if (my $limit = $storage_limits{$storeid}) {
6489                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6490             }
6491
6492             # test if requested format is supported
6493             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6494             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6495             $d->{format} = $defFormat if !$supported;
6496
6497             my $name;
6498             if ($d->{is_cloudinit}) {
6499                 $name = $d->{name};
6500                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6501             }
6502
6503             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6504             print STDERR "new volume ID is '$volid'\n";
6505             $d->{volid} = $volid;
6506
6507             PVE::Storage::activate_volumes($cfg, [$volid]);
6508
6509             my $write_zeros = 1;
6510             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6511                 $write_zeros = 0;
6512             }
6513
6514             if (!$d->{is_cloudinit}) {
6515                 my $path = PVE::Storage::path($cfg, $volid);
6516
6517                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6518
6519                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6520             }
6521             $map->{$virtdev} = $volid;
6522         }
6523
6524         $fh->seek(0, 0) || die "seek failed - $!\n";
6525
6526         my $outfd = new IO::File ($tmpfn, "w") ||
6527             die "unable to write config for VM $vmid\n";
6528
6529         my $cookie = { netcount => 0 };
6530         while (defined(my $line = <$fh>)) {
6531             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6532         }
6533
6534         $fh->close();
6535         $outfd->close();
6536     };
6537
6538     eval {
6539         # enable interrupts
6540         local $SIG{INT} =
6541             local $SIG{TERM} =
6542             local $SIG{QUIT} =
6543             local $SIG{HUP} =
6544             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6545         local $SIG{ALRM} = sub { die "got timeout\n"; };
6546
6547         $oldtimeout = alarm($timeout);
6548
6549         my $parser = sub {
6550             my $line = shift;
6551
6552             print "$line\n";
6553
6554             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6555                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6556                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6557             } elsif ($line =~ m/^CTIME: /) {
6558                 # we correctly received the vma config, so we can disable
6559                 # the timeout now for disk allocation (set to 10 minutes, so
6560                 # that we always timeout if something goes wrong)
6561                 alarm(600);
6562                 &$print_devmap();
6563                 print $fifofh "done\n";
6564                 my $tmp = $oldtimeout || 0;
6565                 $oldtimeout = undef;
6566                 alarm($tmp);
6567                 close($fifofh);
6568             }
6569         };
6570
6571         print "restore vma archive: $dbg_cmdstring\n";
6572         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6573     };
6574     my $err = $@;
6575
6576     alarm($oldtimeout) if $oldtimeout;
6577
6578     my $vollist = [];
6579     foreach my $devname (keys %$devinfo) {
6580         my $volid = $devinfo->{$devname}->{volid};
6581         push @$vollist, $volid if $volid;
6582     }
6583
6584     PVE::Storage::deactivate_volumes($cfg, $vollist);
6585
6586     unlink $mapfifo;
6587
6588     if ($err) {
6589         rmtree $tmpdir;
6590         unlink $tmpfn;
6591
6592         foreach my $devname (keys %$devinfo) {
6593             my $volid = $devinfo->{$devname}->{volid};
6594             next if !$volid;
6595             eval {
6596                 if ($volid =~ m|^/|) {
6597                     unlink $volid || die 'unlink failed\n';
6598                 } else {
6599                     PVE::Storage::vdisk_free($cfg, $volid);
6600                 }
6601                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6602             };
6603             print STDERR "unable to cleanup '$volid' - $@" if $@;
6604         }
6605         die $err;
6606     }
6607
6608     rmtree $tmpdir;
6609
6610     rename($tmpfn, $conffile) ||
6611         die "unable to commit configuration file '$conffile'\n";
6612
6613     PVE::Cluster::cfs_update(); # make sure we read new file
6614
6615     eval { rescan($vmid, 1); };
6616     warn $@ if $@;
6617 }
6618
6619 sub restore_tar_archive {
6620     my ($archive, $vmid, $user, $opts) = @_;
6621
6622     if ($archive ne '-') {
6623         my $firstfile = tar_archive_read_firstfile($archive);
6624         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6625             if $firstfile ne 'qemu-server.conf';
6626     }
6627
6628     my $storecfg = PVE::Storage::config();
6629
6630     # destroy existing data - keep empty config
6631     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6632     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6633
6634     my $tocmd = "/usr/lib/qemu-server/qmextract";
6635
6636     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6637     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6638     $tocmd .= ' --prealloc' if $opts->{prealloc};
6639     $tocmd .= ' --info' if $opts->{info};
6640
6641     # tar option "xf" does not autodetect compression when read from STDIN,
6642     # so we pipe to zcat
6643     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6644         PVE::Tools::shellquote("--to-command=$tocmd");
6645
6646     my $tmpdir = "/var/tmp/vzdumptmp$$";
6647     mkpath $tmpdir;
6648
6649     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6650     local $ENV{VZDUMP_VMID} = $vmid;
6651     local $ENV{VZDUMP_USER} = $user;
6652
6653     my $conffile = PVE::QemuConfig->config_file($vmid);
6654     my $tmpfn = "$conffile.$$.tmp";
6655
6656     # disable interrupts (always do cleanups)
6657     local $SIG{INT} =
6658         local $SIG{TERM} =
6659         local $SIG{QUIT} =
6660         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6661
6662     eval {
6663         # enable interrupts
6664         local $SIG{INT} =
6665             local $SIG{TERM} =
6666             local $SIG{QUIT} =
6667             local $SIG{HUP} =
6668             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6669
6670         if ($archive eq '-') {
6671             print "extracting archive from STDIN\n";
6672             run_command($cmd, input => "<&STDIN");
6673         } else {
6674             print "extracting archive '$archive'\n";
6675             run_command($cmd);
6676         }
6677
6678         return if $opts->{info};
6679
6680         # read new mapping
6681         my $map = {};
6682         my $statfile = "$tmpdir/qmrestore.stat";
6683         if (my $fd = IO::File->new($statfile, "r")) {
6684             while (defined (my $line = <$fd>)) {
6685                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6686                     $map->{$1} = $2 if $1;
6687                 } else {
6688                     print STDERR "unable to parse line in statfile - $line\n";
6689                 }
6690             }
6691             $fd->close();
6692         }
6693
6694         my $confsrc = "$tmpdir/qemu-server.conf";
6695
6696         my $srcfd = new IO::File($confsrc, "r") ||
6697             die "unable to open file '$confsrc'\n";
6698
6699         my $outfd = new IO::File ($tmpfn, "w") ||
6700             die "unable to write config for VM $vmid\n";
6701
6702         my $cookie = { netcount => 0 };
6703         while (defined (my $line = <$srcfd>)) {
6704             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6705         }
6706
6707         $srcfd->close();
6708         $outfd->close();
6709     };
6710     my $err = $@;
6711
6712     if ($err) {
6713
6714         unlink $tmpfn;
6715
6716         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6717
6718         die $err;
6719     }
6720
6721     rmtree $tmpdir;
6722
6723     rename $tmpfn, $conffile ||
6724         die "unable to commit configuration file '$conffile'\n";
6725
6726     PVE::Cluster::cfs_update(); # make sure we read new file
6727
6728     eval { rescan($vmid, 1); };
6729     warn $@ if $@;
6730 };
6731
6732 sub foreach_storage_used_by_vm {
6733     my ($conf, $func) = @_;
6734
6735     my $sidhash = {};
6736
6737     foreach_drive($conf, sub {
6738         my ($ds, $drive) = @_;
6739         return if drive_is_cdrom($drive);
6740
6741         my $volid = $drive->{file};
6742
6743         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6744         $sidhash->{$sid} = $sid if $sid;
6745     });
6746
6747     foreach my $sid (sort keys %$sidhash) {
6748         &$func($sid);
6749     }
6750 }
6751
6752 sub do_snapshots_with_qemu {
6753     my ($storecfg, $volid) = @_;
6754
6755     my $storage_name = PVE::Storage::parse_volume_id($volid);
6756     my $scfg = $storecfg->{ids}->{$storage_name};
6757
6758     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6759         return 1;
6760     }
6761
6762     if ($volid =~ m/\.(qcow2|qed)$/){
6763         return 1;
6764     }
6765
6766     return undef;
6767 }
6768
6769 sub qga_check_running {
6770     my ($vmid, $nowarn) = @_;
6771
6772     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6773     if ($@) {
6774         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6775         return 0;
6776     }
6777     return 1;
6778 }
6779
6780 sub template_create {
6781     my ($vmid, $conf, $disk) = @_;
6782
6783     my $storecfg = PVE::Storage::config();
6784
6785     foreach_drive($conf, sub {
6786         my ($ds, $drive) = @_;
6787
6788         return if drive_is_cdrom($drive);
6789         return if $disk && $ds ne $disk;
6790
6791         my $volid = $drive->{file};
6792         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6793
6794         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6795         $drive->{file} = $voliddst;
6796         $conf->{$ds} = print_drive($vmid, $drive);
6797         PVE::QemuConfig->write_config($vmid, $conf);
6798     });
6799 }
6800
6801 sub convert_iscsi_path {
6802     my ($path) = @_;
6803
6804     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6805         my $portal = $1;
6806         my $target = $2;
6807         my $lun = $3;
6808
6809         my $initiator_name = get_initiator_name();
6810
6811         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6812                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6813     }
6814
6815     die "cannot convert iscsi path '$path', unkown format\n";
6816 }
6817
6818 sub qemu_img_convert {
6819     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6820
6821     my $storecfg = PVE::Storage::config();
6822     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6823     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6824
6825     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6826
6827     my $cachemode;
6828     my $src_path;
6829     my $src_is_iscsi = 0;
6830     my $src_format = 'raw';
6831
6832     if ($src_storeid) {
6833         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6834         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6835         $src_format = qemu_img_format($src_scfg, $src_volname);
6836         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6837         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6838         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6839     } elsif (-f $src_volid) {
6840         $src_path = $src_volid;
6841         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6842             $src_format = $1;
6843         }
6844     }
6845
6846     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6847
6848     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6849     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6850     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6851     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6852
6853     my $cmd = [];
6854     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6855     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6856     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6857     push @$cmd, '-T', $cachemode if defined($cachemode);
6858
6859     if ($src_is_iscsi) {
6860         push @$cmd, '--image-opts';
6861         $src_path = convert_iscsi_path($src_path);
6862     } else {
6863         push @$cmd, '-f', $src_format;
6864     }
6865
6866     if ($dst_is_iscsi) {
6867         push @$cmd, '--target-image-opts';
6868         $dst_path = convert_iscsi_path($dst_path);
6869     } else {
6870         push @$cmd, '-O', $dst_format;
6871     }
6872
6873     push @$cmd, $src_path;
6874
6875     if (!$dst_is_iscsi && $is_zero_initialized) {
6876         push @$cmd, "zeroinit:$dst_path";
6877     } else {
6878         push @$cmd, $dst_path;
6879     }
6880
6881     my $parser = sub {
6882         my $line = shift;
6883         if($line =~ m/\((\S+)\/100\%\)/){
6884             my $percent = $1;
6885             my $transferred = int($size * $percent / 100);
6886             my $remaining = $size - $transferred;
6887
6888             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6889         }
6890
6891     };
6892
6893     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6894     my $err = $@;
6895     die "copy failed: $err" if $err;
6896 }
6897
6898 sub qemu_img_format {
6899     my ($scfg, $volname) = @_;
6900
6901     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6902         return $1;
6903     } else {
6904         return "raw";
6905     }
6906 }
6907
6908 sub qemu_drive_mirror {
6909     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6910
6911     $jobs = {} if !$jobs;
6912
6913     my $qemu_target;
6914     my $format;
6915     $jobs->{"drive-$drive"} = {};
6916
6917     if ($dst_volid =~ /^nbd:/) {
6918         $qemu_target = $dst_volid;
6919         $format = "nbd";
6920     } else {
6921         my $storecfg = PVE::Storage::config();
6922         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6923
6924         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6925
6926         $format = qemu_img_format($dst_scfg, $dst_volname);
6927
6928         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6929
6930         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6931     }
6932
6933     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6934     $opts->{format} = $format if $format;
6935
6936     if (defined($bwlimit)) {
6937         $opts->{speed} = $bwlimit * 1024;
6938         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6939     } else {
6940         print "drive mirror is starting for drive-$drive\n";
6941     }
6942
6943     # if a job already runs for this device we get an error, catch it for cleanup
6944     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6945     if (my $err = $@) {
6946         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6947         warn "$@\n" if $@;
6948         die "mirroring error: $err\n";
6949     }
6950
6951     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6952 }
6953
6954 sub qemu_drive_mirror_monitor {
6955     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6956
6957     eval {
6958         my $err_complete = 0;
6959
6960         while (1) {
6961             die "storage migration timed out\n" if $err_complete > 300;
6962
6963             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6964
6965             my $running_mirror_jobs = {};
6966             foreach my $stat (@$stats) {
6967                 next if $stat->{type} ne 'mirror';
6968                 $running_mirror_jobs->{$stat->{device}} = $stat;
6969             }
6970
6971             my $readycounter = 0;
6972
6973             foreach my $job (keys %$jobs) {
6974
6975                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6976                     print "$job : finished\n";
6977                     delete $jobs->{$job};
6978                     next;
6979                 }
6980
6981                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6982
6983                 my $busy = $running_mirror_jobs->{$job}->{busy};
6984                 my $ready = $running_mirror_jobs->{$job}->{ready};
6985                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6986                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6987                     my $remaining = $total - $transferred;
6988                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6989
6990                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6991                 }
6992
6993                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6994             }
6995
6996             last if scalar(keys %$jobs) == 0;
6997
6998             if ($readycounter == scalar(keys %$jobs)) {
6999                 print "all mirroring jobs are ready \n";
7000                 last if $skipcomplete; #do the complete later
7001
7002                 if ($vmiddst && $vmiddst != $vmid) {
7003                     my $agent_running = $qga && qga_check_running($vmid);
7004                     if ($agent_running) {
7005                         print "freeze filesystem\n";
7006                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7007                     } else {
7008                         print "suspend vm\n";
7009                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7010                     }
7011
7012                     # if we clone a disk for a new target vm, we don't switch the disk
7013                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7014
7015                     if ($agent_running) {
7016                         print "unfreeze filesystem\n";
7017                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7018                     } else {
7019                         print "resume vm\n";
7020                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7021                     }
7022
7023                     last;
7024                 } else {
7025
7026                     foreach my $job (keys %$jobs) {
7027                         # try to switch the disk if source and destination are on the same guest
7028                         print "$job: Completing block job...\n";
7029
7030                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7031                         if ($@ =~ m/cannot be completed/) {
7032                             print "$job: Block job cannot be completed, try again.\n";
7033                             $err_complete++;
7034                         }else {
7035                             print "$job: Completed successfully.\n";
7036                             $jobs->{$job}->{complete} = 1;
7037                         }
7038                     }
7039                 }
7040             }
7041             sleep 1;
7042         }
7043     };
7044     my $err = $@;
7045
7046     if ($err) {
7047         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7048         die "mirroring error: $err";
7049     }
7050
7051 }
7052
7053 sub qemu_blockjobs_cancel {
7054     my ($vmid, $jobs) = @_;
7055
7056     foreach my $job (keys %$jobs) {
7057         print "$job: Cancelling block job\n";
7058         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7059         $jobs->{$job}->{cancel} = 1;
7060     }
7061
7062     while (1) {
7063         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7064
7065         my $running_jobs = {};
7066         foreach my $stat (@$stats) {
7067             $running_jobs->{$stat->{device}} = $stat;
7068         }
7069
7070         foreach my $job (keys %$jobs) {
7071
7072             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7073                 print "$job: Done.\n";
7074                 delete $jobs->{$job};
7075             }
7076         }
7077
7078         last if scalar(keys %$jobs) == 0;
7079
7080         sleep 1;
7081     }
7082 }
7083
7084 sub clone_disk {
7085     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7086         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7087
7088     my $newvolid;
7089
7090     if (!$full) {
7091         print "create linked clone of drive $drivename ($drive->{file})\n";
7092         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7093         push @$newvollist, $newvolid;
7094     } else {
7095
7096         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7097         $storeid = $storage if $storage;
7098
7099         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7100         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7101
7102         print "create full clone of drive $drivename ($drive->{file})\n";
7103         my $name = undef;
7104         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7105         push @$newvollist, $newvolid;
7106
7107         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7108
7109         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7110         if (!$running || $snapname) {
7111             # TODO: handle bwlimits
7112             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7113         } else {
7114
7115             my $kvmver = get_running_qemu_version ($vmid);
7116             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7117                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7118                     if $drive->{iothread};
7119             }
7120
7121             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7122         }
7123     }
7124
7125     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7126
7127     my $disk = $drive;
7128     $disk->{format} = undef;
7129     $disk->{file} = $newvolid;
7130     $disk->{size} = $size;
7131
7132     return $disk;
7133 }
7134
7135 # this only works if VM is running
7136 sub get_current_qemu_machine {
7137     my ($vmid) = @_;
7138
7139     my $cmd = { execute => 'query-machines', arguments => {} };
7140     my $res = vm_qmp_command($vmid, $cmd);
7141
7142     my ($current, $default);
7143     foreach my $e (@$res) {
7144         $default = $e->{name} if $e->{'is-default'};
7145         $current = $e->{name} if $e->{'is-current'};
7146     }
7147
7148     # fallback to the default machine if current is not supported by qemu
7149     return $current || $default || 'pc';
7150 }
7151
7152 sub get_running_qemu_version {
7153     my ($vmid) = @_;
7154     my $cmd = { execute => 'query-version', arguments => {} };
7155     my $res = vm_qmp_command($vmid, $cmd);
7156     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7157 }
7158
7159 sub qemu_machine_feature_enabled {
7160     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7161
7162     my $current_major;
7163     my $current_minor;
7164
7165     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7166
7167         $current_major = $3;
7168         $current_minor = $4;
7169
7170     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7171
7172         $current_major = $1;
7173         $current_minor = $2;
7174     }
7175
7176     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7177 }
7178
7179 # gets in pairs the versions you want to compares, i.e.:
7180 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7181 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7182 sub version_cmp {
7183     my @versions = @_;
7184
7185     my $size = scalar(@versions);
7186
7187     return 0 if $size == 0;
7188     die "cannot compare odd count of versions" if $size & 1;
7189
7190     for (my $i = 0; $i < $size; $i += 2) {
7191         my ($a, $b) = splice(@versions, 0, 2);
7192         $a //= 0;
7193         $b //= 0;
7194
7195         return 1 if $a > $b;
7196         return -1 if $a < $b;
7197     }
7198     return 0;
7199 }
7200
7201 sub runs_at_least_qemu_version {
7202     my ($vmid, $major, $minor, $extra) = @_;
7203
7204     my $v = eval { PVE::QemuServer::vm_qmp_command($vmid, { execute => 'query-version' }) } // {};
7205     $v = $v->{qemu};
7206
7207     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7208 }
7209
7210 sub qemu_machine_pxe {
7211     my ($vmid, $conf) = @_;
7212
7213     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7214
7215     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7216         $machine .= '.pxe';
7217     }
7218
7219     return $machine;
7220 }
7221
7222 sub qemu_use_old_bios_files {
7223     my ($machine_type) = @_;
7224
7225     return if !$machine_type;
7226
7227     my $use_old_bios_files = undef;
7228
7229     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7230         $machine_type = $1;
7231         $use_old_bios_files = 1;
7232     } else {
7233         my $kvmver = kvm_user_version();
7234         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7235         # load new efi bios files on migration. So this hack is required to allow
7236         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7237         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7238         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7239     }
7240
7241     return ($use_old_bios_files, $machine_type);
7242 }
7243
7244 sub create_efidisk($$$$$) {
7245     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7246
7247     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7248     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7249
7250     my $vars_size_b = -s $ovmf_vars;
7251     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7252     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7253     PVE::Storage::activate_volumes($storecfg, [$volid]);
7254
7255     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7256
7257     return ($volid, $vars_size);
7258 }
7259
7260 sub vm_iothreads_list {
7261     my ($vmid) = @_;
7262
7263     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7264
7265     my $iothreads = {};
7266     foreach my $iothread (@$res) {
7267         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7268     }
7269
7270     return $iothreads;
7271 }
7272
7273 sub scsihw_infos {
7274     my ($conf, $drive) = @_;
7275
7276     my $maxdev = 0;
7277
7278     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7279         $maxdev = 7;
7280     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7281         $maxdev = 1;
7282     } else {
7283         $maxdev = 256;
7284     }
7285
7286     my $controller = int($drive->{index} / $maxdev);
7287     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7288
7289     return ($maxdev, $controller, $controller_prefix);
7290 }
7291
7292 sub add_hyperv_enlightenments {
7293     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7294
7295     return if $winversion < 6;
7296     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7297
7298     if ($gpu_passthrough || defined($hv_vendor_id)) {
7299         $hv_vendor_id //= 'proxmox';
7300         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7301     }
7302
7303     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7304         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7305         push @$cpuFlags , 'hv_vapic';
7306         push @$cpuFlags , 'hv_time';
7307     } else {
7308         push @$cpuFlags , 'hv_spinlocks=0xffff';
7309     }
7310
7311     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7312         push @$cpuFlags , 'hv_reset';
7313         push @$cpuFlags , 'hv_vpindex';
7314         push @$cpuFlags , 'hv_runtime';
7315     }
7316
7317     if ($winversion >= 7) {
7318         push @$cpuFlags , 'hv_relaxed';
7319
7320         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7321             push @$cpuFlags , 'hv_synic';
7322             push @$cpuFlags , 'hv_stimer';
7323         }
7324
7325         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7326             push @$cpuFlags , 'hv_ipi';
7327         }
7328     }
7329 }
7330
7331 sub windows_version {
7332     my ($ostype) = @_;
7333
7334     return 0 if !$ostype;
7335
7336     my $winversion = 0;
7337
7338     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7339         $winversion = 5;
7340     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7341         $winversion = 6;
7342     } elsif ($ostype =~ m/^win(\d+)$/) {
7343         $winversion = $1;
7344     }
7345
7346     return $winversion;
7347 }
7348
7349 sub resolve_dst_disk_format {
7350         my ($storecfg, $storeid, $src_volname, $format) = @_;
7351         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7352
7353         if (!$format) {
7354             # if no target format is specified, use the source disk format as hint
7355             if ($src_volname) {
7356                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7357                 $format = qemu_img_format($scfg, $src_volname);
7358             } else {
7359                 return $defFormat;
7360             }
7361         }
7362
7363         # test if requested format is supported - else use default
7364         my $supported = grep { $_ eq $format } @$validFormats;
7365         $format = $defFormat if !$supported;
7366         return $format;
7367 }
7368
7369 sub resolve_first_disk {
7370     my $conf = shift;
7371     my @disks = PVE::QemuServer::valid_drive_names();
7372     my $firstdisk;
7373     foreach my $ds (reverse @disks) {
7374         next if !$conf->{$ds};
7375         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7376         next if PVE::QemuServer::drive_is_cdrom($disk);
7377         $firstdisk = $ds;
7378     }
7379     return $firstdisk;
7380 }
7381
7382 sub generate_uuid {
7383     my ($uuid, $uuid_str);
7384     UUID::generate($uuid);
7385     UUID::unparse($uuid, $uuid_str);
7386     return $uuid_str;
7387 }
7388
7389 sub generate_smbios1_uuid {
7390     return "uuid=".generate_uuid();
7391 }
7392
7393 sub nbd_stop {
7394     my ($vmid) = @_;
7395
7396     vm_mon_cmd($vmid, 'nbd-server-stop');
7397 }
7398
7399 sub create_reboot_request {
7400     my ($vmid) = @_;
7401     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7402         or die "failed to create reboot trigger file: $!\n";
7403     close($fh);
7404 }
7405
7406 sub clear_reboot_request {
7407     my ($vmid) = @_;
7408     my $path = "/run/qemu-server/$vmid.reboot";
7409     my $res = 0;
7410
7411     $res = unlink($path);
7412     die "could not remove reboot request for $vmid: $!"
7413         if !$res && $! != POSIX::ENOENT;
7414
7415     return $res;
7416 }
7417
7418 # bash completion helper
7419
7420 sub complete_backup_archives {
7421     my ($cmdname, $pname, $cvalue) = @_;
7422
7423     my $cfg = PVE::Storage::config();
7424
7425     my $storeid;
7426
7427     if ($cvalue =~ m/^([^:]+):/) {
7428         $storeid = $1;
7429     }
7430
7431     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7432
7433     my $res = [];
7434     foreach my $id (keys %$data) {
7435         foreach my $item (@{$data->{$id}}) {
7436             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7437             push @$res, $item->{volid} if defined($item->{volid});
7438         }
7439     }
7440
7441     return $res;
7442 }
7443
7444 my $complete_vmid_full = sub {
7445     my ($running) = @_;
7446
7447     my $idlist = vmstatus();
7448
7449     my $res = [];
7450
7451     foreach my $id (keys %$idlist) {
7452         my $d = $idlist->{$id};
7453         if (defined($running)) {
7454             next if $d->{template};
7455             next if $running && $d->{status} ne 'running';
7456             next if !$running && $d->{status} eq 'running';
7457         }
7458         push @$res, $id;
7459
7460     }
7461     return $res;
7462 };
7463
7464 sub complete_vmid {
7465     return &$complete_vmid_full();
7466 }
7467
7468 sub complete_vmid_stopped {
7469     return &$complete_vmid_full(0);
7470 }
7471
7472 sub complete_vmid_running {
7473     return &$complete_vmid_full(1);
7474 }
7475
7476 sub complete_storage {
7477
7478     my $cfg = PVE::Storage::config();
7479     my $ids = $cfg->{ids};
7480
7481     my $res = [];
7482     foreach my $sid (keys %$ids) {
7483         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7484         next if !$ids->{$sid}->{content}->{images};
7485         push @$res, $sid;
7486     }
7487
7488     return $res;
7489 }
7490
7491 1;