PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs',
 182     'aes'
 183 );
 184 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 185
 186 my $cpu_fmt = {
 187     cputype => {
 188         description => "Emulated CPU type.",
 189         type => 'string',
 190         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 191         default => 'kvm64',
 192         default_key => 1,
 193     },
 194     hidden => {
 195         description => "Do not identify as a KVM virtual machine.",
 196         type => 'boolean',
 197         optional => 1,
 198         default => 0
 199     },
 200     'hv-vendor-id' => {
 201         type => 'string',
 202         pattern => qr/[a-zA-Z0-9]{1,12}/,
 203         format_description => 'vendor-id',
 204         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 205         optional => 1,
 206     },
 207     flags => {
 208         description => "List of additional CPU flags separated by ';'."
 209                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 210                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 211         format_description => '+FLAG[;-FLAG...]',
 212         type => 'string',
 213         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 214         optional => 1,
 215     },
 216 };
 217
 218 my $watchdog_fmt = {
 219     model => {
 220         default_key => 1,
 221         type => 'string',
 222         enum => [qw(i6300esb ib700)],
 223         description => "Watchdog type to emulate.",
 224         default => 'i6300esb',
 225         optional => 1,
 226     },
 227     action => {
 228         type => 'string',
 229         enum => [qw(reset shutdown poweroff pause debug none)],
 230         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 231         optional => 1,
 232     },
 233 };
 234 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 235
 236 my $agent_fmt = {
 237     enabled => {
 238         description => "Enable/disable Qemu GuestAgent.",
 239         type => 'boolean',
 240         default => 0,
 241         default_key => 1,
 242     },
 243     fstrim_cloned_disks => {
 244         description => "Run fstrim after cloning/moving a disk.",
 245         type => 'boolean',
 246         optional => 1,
 247         default => 0
 248     },
 249 };
 250
 251 my $vga_fmt = {
 252     type => {
 253         description => "Select the VGA type.",
 254         type => 'string',
 255         default => 'std',
 256         optional => 1,
 257         default_key => 1,
 258         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 259     },
 260     memory => {
 261         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 262         type => 'integer',
 263         optional => 1,
 264         minimum => 4,
 265         maximum => 512,
 266     },
 267 };
 268
 269 my $ivshmem_fmt = {
 270     size => {
 271         type => 'integer',
 272         minimum => 1,
 273         description => "The size of the file in MB.",
 274     },
 275     name => {
 276         type => 'string',
 277         pattern => '[a-zA-Z0-9\-]+',
 278         optional => 1,
 279         format_description => 'string',
 280         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 281     },
 282 };
 283
 284 my $audio_fmt = {
 285     device => {
 286         type => 'string',
 287         enum => [qw(ich9-intel-hda intel-hda AC97)],
 288         description =>  "Configure an audio device."
 289     },
 290     driver =>  {
 291         type => 'string',
 292         enum => ['spice'],
 293         default => 'spice',
 294         optional => 1,
 295         description => "Driver backend for the audio device."
 296     },
 297 };
 298
 299 my $spice_enhancements_fmt = {
 300     foldersharing => {
 301         type => 'boolean',
 302         optional => 1,
 303         default => '0',
 304         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 305     },
 306     videostreaming =>  {
 307         type => 'string',
 308         enum => ['off', 'all', 'filter'],
 309         default => 'off',
 310         optional => 1,
 311         description => "Enable video streaming. Uses compression for detected video streams."
 312     },
 313 };
 314
 315 my $confdesc = {
 316     onboot => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Specifies whether a VM will be started during system bootup.",
 320         default => 0,
 321     },
 322     autostart => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Automatic restart after crash (currently ignored).",
 326         default => 0,
 327     },
 328     hotplug => {
 329         optional => 1,
 330         type => 'string', format => 'pve-hotplug-features',
 331         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 332         default => 'network,disk,usb',
 333     },
 334     reboot => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 338         default => 1,
 339     },
 340     lock => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Lock/unlock the VM.",
 344         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 345     },
 346     cpulimit => {
 347         optional => 1,
 348         type => 'number',
 349         description => "Limit of CPU usage.",
 350         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 351         minimum => 0,
 352         maximum => 128,
 353         default => 0,
 354     },
 355     cpuunits => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "CPU weight for a VM.",
 359         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 360         minimum => 2,
 361         maximum => 262144,
 362         default => 1024,
 363     },
 364     memory => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 368         minimum => 16,
 369         default => 512,
 370     },
 371     balloon => {
 372         optional => 1,
 373         type => 'integer',
 374         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 375         minimum => 0,
 376     },
 377     shares => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 389                        "It should not be necessary to set it.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 409     },
 410     ostype => {
 411         optional => 1,
 412         type => 'string',
 413         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 414         description => "Specify guest operating system.",
 415         verbose_description => <<EODESC,
 416 Specify guest operating system. This is used to enable special
 417 optimization/features for specific operating systems:
 418
 419 [horizontal]
 420 other;; unspecified OS
 421 wxp;; Microsoft Windows XP
 422 w2k;; Microsoft Windows 2000
 423 w2k3;; Microsoft Windows 2003
 424 w2k8;; Microsoft Windows 2008
 425 wvista;; Microsoft Windows Vista
 426 win7;; Microsoft Windows 7
 427 win8;; Microsoft Windows 8/2012/2012r2
 428 win10;; Microsoft Windows 10/2016
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6/3.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string',
 437         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 438         pattern => '[acdn]{1,4}',
 439         default => 'cdn',
 440     },
 441     bootdisk => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-bootdisk',
 444         description => "Enable booting from specified disk.",
 445         pattern => '(ide|sata|scsi|virtio)\d+',
 446     },
 447     smp => {
 448         optional => 1,
 449         type => 'integer',
 450         description => "The number of CPUs. Please use option -sockets instead.",
 451         minimum => 1,
 452         default => 1,
 453     },
 454     sockets => {
 455         optional => 1,
 456         type => 'integer',
 457         description => "The number of CPU sockets.",
 458         minimum => 1,
 459         default => 1,
 460     },
 461     cores => {
 462         optional => 1,
 463         type => 'integer',
 464         description => "The number of cores per socket.",
 465         minimum => 1,
 466         default => 1,
 467     },
 468     numa => {
 469         optional => 1,
 470         type => 'boolean',
 471         description => "Enable/disable NUMA.",
 472         default => 0,
 473     },
 474     hugepages => {
 475         optional => 1,
 476         type => 'string',
 477         description => "Enable/disable hugepages memory.",
 478         enum => [qw(any 2 1024)],
 479     },
 480     vcpus => {
 481         optional => 1,
 482         type => 'integer',
 483         description => "Number of hotplugged vcpus.",
 484         minimum => 1,
 485         default => 0,
 486     },
 487     acpi => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable ACPI.",
 491         default => 1,
 492     },
 493     agent => {
 494         optional => 1,
 495         description => "Enable/disable Qemu GuestAgent and its properties.",
 496         type => 'string',
 497         format => $agent_fmt,
 498     },
 499     kvm => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Enable/disable KVM hardware virtualization.",
 503         default => 1,
 504     },
 505     tdf => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable time drift fix.",
 509         default => 0,
 510     },
 511     localtime => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 515     },
 516     freeze => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 520     },
 521     vga => {
 522         optional => 1,
 523         type => 'string', format => $vga_fmt,
 524         description => "Configure the VGA hardware.",
 525         verbose_description => "Configure the VGA Hardware. If you want to use ".
 526             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 527             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 528             "is 'std' for all OS types besides some Windows versions (XP and " .
 529             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 530             "display server. For win* OS you can select how many independent " .
 531             "displays you want, Linux guests can add displays them self.\n".
 532             "You can also run without any graphic card, using a serial device as terminal.",
 533     },
 534     watchdog => {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-watchdog',
 537         description => "Create a virtual hardware watchdog device.",
 538         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 539             " (by a guest action), the watchdog must be periodically polled " .
 540             "by an agent inside the guest or else the watchdog will reset " .
 541             "the guest (or execute the respective action specified)",
 542     },
 543     startdate => {
 544         optional => 1,
 545         type => 'string',
 546         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 547         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 548         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 549         default => 'now',
 550     },
 551     startup =>  get_standard_option('pve-startup-order'),
 552     template => {
 553         optional => 1,
 554         type => 'boolean',
 555         description => "Enable/disable Template.",
 556         default => 0,
 557     },
 558     args => {
 559         optional => 1,
 560         type => 'string',
 561         description => "Arbitrary arguments passed to kvm.",
 562         verbose_description => <<EODESCR,
 563 Arbitrary arguments passed to kvm, for example:
 564
 565 args: -no-reboot -no-hpet
 566
 567 NOTE: this option is for experts only.
 568 EODESCR
 569     },
 570     tablet => {
 571         optional => 1,
 572         type => 'boolean',
 573         default => 1,
 574         description => "Enable/disable the USB tablet device.",
 575         verbose_description => "Enable/disable the USB tablet device. This device is " .
 576             "usually needed to allow absolute mouse positioning with VNC. " .
 577             "Else the mouse runs out of sync with normal VNC clients. " .
 578             "If you're running lots of console-only guests on one host, " .
 579             "you may consider disabling this to save some context switches. " .
 580             "This is turned off by default if you use spice (-vga=qxl).",
 581     },
 582     migrate_speed => {
 583         optional => 1,
 584         type => 'integer',
 585         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 586         minimum => 0,
 587         default => 0,
 588     },
 589     migrate_downtime => {
 590         optional => 1,
 591         type => 'number',
 592         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 593         minimum => 0,
 594         default => 0.1,
 595     },
 596     cdrom => {
 597         optional => 1,
 598         type => 'string', format => 'pve-qm-ide',
 599         typetext => '<volume>',
 600         description => "This is an alias for option -ide2",
 601     },
 602     cpu => {
 603         optional => 1,
 604         description => "Emulated CPU type.",
 605         type => 'string',
 606         format => $cpu_fmt,
 607     },
 608     parent => get_standard_option('pve-snapshot-name', {
 609         optional => 1,
 610         description => "Parent snapshot name. This is used internally, and should not be modified.",
 611     }),
 612     snaptime => {
 613         optional => 1,
 614         description => "Timestamp for snapshots.",
 615         type => 'integer',
 616         minimum => 0,
 617     },
 618     vmstate => {
 619         optional => 1,
 620         type => 'string', format => 'pve-volume-id',
 621         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 622     },
 623     vmstatestorage => get_standard_option('pve-storage-id', {
 624         description => "Default storage for VM state volumes/files.",
 625         optional => 1,
 626     }),
 627     runningmachine => get_standard_option('pve-qemu-machine', {
 628         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 629     }),
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 662             " 128-bit integer value identifier to the guest OS. This allows to".
 663             " notify the guest operating system when the virtual machine is".
 664             " executed with a different configuration (e.g. snapshot execution".
 665             " or creation from a template). The guest operating system notices".
 666             " the change, and is then able to react as appropriate by marking".
 667             " its copies of distributed databases as dirty, re-initializing its".
 668             " random number generator, etc.\n".
 669             "Note that auto-creation only works when done throug API/CLI create".
 670             " or update methods, but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698 };
 699
 700 my $cicustom_fmt = {
 701     meta => {
 702         type => 'string',
 703         optional => 1,
 704         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     network => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 712         format => 'pve-volume-id',
 713         format_description => 'volume',
 714     },
 715     user => {
 716         type => 'string',
 717         optional => 1,
 718         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 719         format => 'pve-volume-id',
 720         format_description => 'volume',
 721     },
 722 };
 723 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 724
 725 my $confdesc_cloudinit = {
 726     citype => {
 727         optional => 1,
 728         type => 'string',
 729         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 730         enum => ['configdrive2', 'nocloud'],
 731     },
 732     ciuser => {
 733         optional => 1,
 734         type => 'string',
 735         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 736     },
 737     cipassword => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 741     },
 742     cicustom => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_IDE_DISKS = 4;
 788 my $MAX_SCSI_DISKS = 14;
 789 my $MAX_VIRTIO_DISKS = 16;
 790 my $MAX_SATA_DISKS = 6;
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_UNUSED_DISKS = 256;
 794 my $MAX_HOSTPCI_DEVICES = 16;
 795 my $MAX_SERIAL_PORTS = 4;
 796 my $MAX_PARALLEL_PORTS = 3;
 797 my $MAX_NUMA = 8;
 798
 799 my $numa_fmt = {
 800     cpus => {
 801         type => "string",
 802         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 803         description => "CPUs accessing this NUMA node.",
 804         format_description => "id[-id];...",
 805     },
 806     memory => {
 807         type => "number",
 808         description => "Amount of memory this NUMA node provides.",
 809         optional => 1,
 810     },
 811     hostnodes => {
 812         type => "string",
 813         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 814         description => "Host NUMA nodes to use.",
 815         format_description => "id[-id];...",
 816         optional => 1,
 817     },
 818     policy => {
 819         type => 'string',
 820         enum => [qw(preferred bind interleave)],
 821         description => "NUMA allocation policy.",
 822         optional => 1,
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 826 my $numadesc = {
 827     optional => 1,
 828     type => 'string', format => $numa_fmt,
 829     description => "NUMA topology.",
 830 };
 831 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 832
 833 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 834     $confdesc->{"numa$i"} = $numadesc;
 835 }
 836
 837 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 838                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 839                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 840 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 841
 842 my $net_fmt_bridge_descr = <<__EOD__;
 843 Bridge to attach the network device to. The Proxmox VE standard bridge
 844 is called 'vmbr0'.
 845
 846 If you do not specify a bridge, we create a kvm user (NATed) network
 847 device, which provides DHCP and DNS services. The following addresses
 848 are used:
 849
 850  10.0.2.2   Gateway
 851  10.0.2.3   DNS Server
 852  10.0.2.4   SMB Server
 853
 854 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 855 __EOD__
 856
 857 my $net_fmt = {
 858     macaddr  => get_standard_option('mac-addr', {
 859         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 860     }),
 861     model => {
 862         type => 'string',
 863         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 864         enum => $nic_model_list,
 865         default_key => 1,
 866     },
 867     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 868     bridge => {
 869         type => 'string',
 870         description => $net_fmt_bridge_descr,
 871         format_description => 'bridge',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909 };
 910
 911 my $netdesc = {
 912     optional => 1,
 913     type => 'string', format => $net_fmt,
 914     description => "Specify network devices.",
 915 };
 916
 917 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 918
 919 my $ipconfig_fmt = {
 920     ip => {
 921         type => 'string',
 922         format => 'pve-ipv4-config',
 923         format_description => 'IPv4Format/CIDR',
 924         description => 'IPv4 address in CIDR format.',
 925         optional => 1,
 926         default => 'dhcp',
 927     },
 928     gw => {
 929         type => 'string',
 930         format => 'ipv4',
 931         format_description => 'GatewayIPv4',
 932         description => 'Default gateway for IPv4 traffic.',
 933         optional => 1,
 934         requires => 'ip',
 935     },
 936     ip6 => {
 937         type => 'string',
 938         format => 'pve-ipv6-config',
 939         format_description => 'IPv6Format/CIDR',
 940         description => 'IPv6 address in CIDR format.',
 941         optional => 1,
 942         default => 'dhcp',
 943     },
 944     gw6 => {
 945         type => 'string',
 946         format => 'ipv6',
 947         format_description => 'GatewayIPv6',
 948         description => 'Default gateway for IPv6 traffic.',
 949         optional => 1,
 950         requires => 'ip6',
 951     },
 952 };
 953 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 954 my $ipconfigdesc = {
 955     optional => 1,
 956     type => 'string', format => 'pve-qm-ipconfig',
 957     description => <<'EODESCR',
 958 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 959
 960 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 961
 962 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 963 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 964
 965 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 966 EODESCR
 967 };
 968 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 969
 970 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 971     $confdesc->{"net$i"} = $netdesc;
 972     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 973 }
 974
 975 foreach my $key (keys %$confdesc_cloudinit) {
 976     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 977 }
 978
 979 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 980 sub verify_volume_id_or_qm_path {
 981     my ($volid, $noerr) = @_;
 982
 983     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 984         return $volid;
 985     }
 986
 987     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 988     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 989     if ($@) {
 990         return undef if $noerr;
 991         die $@;
 992     }
 993     return $volid;
 994 }
 995
 996 my $drivename_hash;
 997
 998 my %drivedesc_base = (
 999     volume => { alias => 'file' },
1000     file => {
1001         type => 'string',
1002         format => 'pve-volume-id-or-qm-path',
1003         default_key => 1,
1004         format_description => 'volume',
1005         description => "The drive's backing volume.",
1006     },
1007     media => {
1008         type => 'string',
1009         enum => [qw(cdrom disk)],
1010         description => "The drive's media type.",
1011         default => 'disk',
1012         optional => 1
1013     },
1014     cyls => {
1015         type => 'integer',
1016         description => "Force the drive's physical geometry to have a specific cylinder count.",
1017         optional => 1
1018     },
1019     heads => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific head count.",
1022         optional => 1
1023     },
1024     secs => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific sector count.",
1027         optional => 1
1028     },
1029     trans => {
1030         type => 'string',
1031         enum => [qw(none lba auto)],
1032         description => "Force disk geometry bios translation mode.",
1033         optional => 1,
1034     },
1035     snapshot => {
1036         type => 'boolean',
1037         description => "Controls qemu's snapshot mode feature."
1038             . " If activated, changes made to the disk are temporary and will"
1039             . " be discarded when the VM is shutdown.",
1040         optional => 1,
1041     },
1042     cache => {
1043         type => 'string',
1044         enum => [qw(none writethrough writeback unsafe directsync)],
1045         description => "The drive's cache mode",
1046         optional => 1,
1047     },
1048     format => get_standard_option('pve-qm-image-format'),
1049     size => {
1050         type => 'string',
1051         format => 'disk-size',
1052         format_description => 'DiskSize',
1053         description => "Disk size. This is purely informational and has no effect.",
1054         optional => 1,
1055     },
1056     backup => {
1057         type => 'boolean',
1058         description => "Whether the drive should be included when making backups.",
1059         optional => 1,
1060     },
1061     replicate => {
1062         type => 'boolean',
1063         description => 'Whether the drive should considered for replication jobs.',
1064         optional => 1,
1065         default => 1,
1066     },
1067     rerror => {
1068         type => 'string',
1069         enum => [qw(ignore report stop)],
1070         description => 'Read error action.',
1071         optional => 1,
1072     },
1073     werror => {
1074         type => 'string',
1075         enum => [qw(enospc ignore report stop)],
1076         description => 'Write error action.',
1077         optional => 1,
1078     },
1079     aio => {
1080         type => 'string',
1081         enum => [qw(native threads)],
1082         description => 'AIO type to use.',
1083         optional => 1,
1084     },
1085     discard => {
1086         type => 'string',
1087         enum => [qw(ignore on)],
1088         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1089         optional => 1,
1090     },
1091     detect_zeroes => {
1092         type => 'boolean',
1093         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1094         optional => 1,
1095     },
1096     serial => {
1097         type => 'string',
1098         format => 'urlencoded',
1099         format_description => 'serial',
1100         maxLength => 20*3, # *3 since it's %xx url enoded
1101         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1102         optional => 1,
1103     },
1104     shared => {
1105         type => 'boolean',
1106         description => 'Mark this locally-managed volume as available on all nodes',
1107         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1108         optional => 1,
1109         default => 0,
1110     }
1111 );
1112
1113 my %iothread_fmt = ( iothread => {
1114         type => 'boolean',
1115         description => "Whether to use iothreads for this drive",
1116         optional => 1,
1117 });
1118
1119 my %model_fmt = (
1120     model => {
1121         type => 'string',
1122         format => 'urlencoded',
1123         format_description => 'model',
1124         maxLength => 40*3, # *3 since it's %xx url enoded
1125         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1126         optional => 1,
1127     },
1128 );
1129
1130 my %queues_fmt = (
1131     queues => {
1132         type => 'integer',
1133         description => "Number of queues.",
1134         minimum => 2,
1135         optional => 1
1136     }
1137 );
1138
1139 my %scsiblock_fmt = (
1140     scsiblock => {
1141         type => 'boolean',
1142         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1143         optional => 1,
1144         default => 0,
1145     },
1146 );
1147
1148 my %ssd_fmt = (
1149     ssd => {
1150         type => 'boolean',
1151         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1152         optional => 1,
1153     },
1154 );
1155
1156 my %wwn_fmt = (
1157     wwn => {
1158         type => 'string',
1159         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1160         format_description => 'wwn',
1161         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1162         optional => 1,
1163     },
1164 );
1165
1166 my $add_throttle_desc = sub {
1167     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1168     my $d = {
1169         type => $type,
1170         format_description => $unit,
1171         description => "Maximum $what in $longunit.",
1172         optional => 1,
1173     };
1174     $d->{minimum} = $minimum if defined($minimum);
1175     $drivedesc_base{$key} = $d;
1176 };
1177 # throughput: (leaky bucket)
1178 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1179 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1180 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1181 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1182 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1183 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1184 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1185 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1186 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1187
1188 # pools: (pool of IO before throttling starts taking effect)
1189 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1190 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1192 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1193 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1194 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1195
1196 # burst lengths
1197 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1198 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1199 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1200 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203
1204 # legacy support
1205 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1206 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1207 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1208 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1209
1210 my $ide_fmt = {
1211     %drivedesc_base,
1212     %model_fmt,
1213     %ssd_fmt,
1214     %wwn_fmt,
1215 };
1216 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1217
1218 my $idedesc = {
1219     optional => 1,
1220     type => 'string', format => $ide_fmt,
1221     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1222 };
1223 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1224
1225 my $scsi_fmt = {
1226     %drivedesc_base,
1227     %iothread_fmt,
1228     %queues_fmt,
1229     %scsiblock_fmt,
1230     %ssd_fmt,
1231     %wwn_fmt,
1232 };
1233 my $scsidesc = {
1234     optional => 1,
1235     type => 'string', format => $scsi_fmt,
1236     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1237 };
1238 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1239
1240 my $sata_fmt = {
1241     %drivedesc_base,
1242     %ssd_fmt,
1243     %wwn_fmt,
1244 };
1245 my $satadesc = {
1246     optional => 1,
1247     type => 'string', format => $sata_fmt,
1248     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1251
1252 my $virtio_fmt = {
1253     %drivedesc_base,
1254     %iothread_fmt,
1255 };
1256 my $virtiodesc = {
1257     optional => 1,
1258     type => 'string', format => $virtio_fmt,
1259     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1260 };
1261 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1262
1263 my $alldrive_fmt = {
1264     %drivedesc_base,
1265     %iothread_fmt,
1266     %model_fmt,
1267     %queues_fmt,
1268     %scsiblock_fmt,
1269     %ssd_fmt,
1270     %wwn_fmt,
1271 };
1272
1273 my $efidisk_fmt = {
1274     volume => { alias => 'file' },
1275     file => {
1276         type => 'string',
1277         format => 'pve-volume-id-or-qm-path',
1278         default_key => 1,
1279         format_description => 'volume',
1280         description => "The drive's backing volume.",
1281     },
1282     format => get_standard_option('pve-qm-image-format'),
1283     size => {
1284         type => 'string',
1285         format => 'disk-size',
1286         format_description => 'DiskSize',
1287         description => "Disk size. This is purely informational and has no effect.",
1288         optional => 1,
1289     },
1290 };
1291
1292 my $efidisk_desc = {
1293     optional => 1,
1294     type => 'string', format => $efidisk_fmt,
1295     description => "Configure a Disk for storing EFI vars",
1296 };
1297
1298 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1299
1300 my $usb_fmt = {
1301     host => {
1302         default_key => 1,
1303         type => 'string', format => 'pve-qm-usb-device',
1304         format_description => 'HOSTUSBDEVICE|spice',
1305         description => <<EODESCR,
1306 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1307
1308  'bus-port(.port)*' (decimal numbers) or
1309  'vendor_id:product_id' (hexadeciaml numbers) or
1310  'spice'
1311
1312 You can use the 'lsusb -t' command to list existing usb devices.
1313
1314 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1315
1316 The value 'spice' can be used to add a usb redirection devices for spice.
1317 EODESCR
1318     },
1319     usb3 => {
1320         optional => 1,
1321         type => 'boolean',
1322         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1323         default => 0,
1324     },
1325 };
1326
1327 my $usbdesc = {
1328     optional => 1,
1329     type => 'string', format => $usb_fmt,
1330     description => "Configure an USB device (n is 0 to 4).",
1331 };
1332 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1333
1334 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1335 my $hostpci_fmt = {
1336     host => {
1337         default_key => 1,
1338         type => 'string',
1339         pattern => qr/$PCIRE(;$PCIRE)*/,
1340         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1341         description => <<EODESCR,
1342 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1343 of PCI virtual functions of the host. HOSTPCIID syntax is:
1344
1345 'bus:dev.func' (hexadecimal numbers)
1346
1347 You can us the 'lspci' command to list existing PCI devices.
1348 EODESCR
1349     },
1350     rombar => {
1351         type => 'boolean',
1352         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1353         optional => 1,
1354         default => 1,
1355     },
1356     romfile => {
1357         type => 'string',
1358         pattern => '[^,;]+',
1359         format_description => 'string',
1360         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1361         optional => 1,
1362     },
1363     pcie => {
1364         type => 'boolean',
1365         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1366         optional => 1,
1367         default => 0,
1368     },
1369     'x-vga' => {
1370         type => 'boolean',
1371         description =>  "Enable vfio-vga device support.",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'mdev' => {
1376         type => 'string',
1377         format_description => 'string',
1378         pattern => '[^/\.:]+',
1379         optional => 1,
1380         description => <<EODESCR
1381 The type of mediated device to use.
1382 An instance of this type will be created on startup of the VM and
1383 will be cleaned up when the VM stops.
1384 EODESCR
1385     }
1386 };
1387 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1388
1389 my $hostpcidesc = {
1390         optional => 1,
1391         type => 'string', format => 'pve-qm-hostpci',
1392         description => "Map host PCI devices into guest.",
1393         verbose_description =>  <<EODESCR,
1394 Map host PCI devices into guest.
1395
1396 NOTE: This option allows direct access to host hardware. So it is no longer
1397 possible to migrate such machines - use with special care.
1398
1399 CAUTION: Experimental! User reported problems with this option.
1400 EODESCR
1401 };
1402 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1403
1404 my $serialdesc = {
1405         optional => 1,
1406         type => 'string',
1407         pattern => '(/dev/.+|socket)',
1408         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1409         verbose_description =>  <<EODESCR,
1410 Create a serial device inside the VM (n is 0 to 3), and pass through a
1411 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1412 host side (use 'qm terminal' to open a terminal connection).
1413
1414 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1415
1416 CAUTION: Experimental! User reported problems with this option.
1417 EODESCR
1418 };
1419
1420 my $paralleldesc= {
1421         optional => 1,
1422         type => 'string',
1423         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1424         description =>  "Map host parallel devices (n is 0 to 2).",
1425         verbose_description =>  <<EODESCR,
1426 Map host parallel devices (n is 0 to 2).
1427
1428 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1429
1430 CAUTION: Experimental! User reported problems with this option.
1431 EODESCR
1432 };
1433
1434 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1435     $confdesc->{"parallel$i"} = $paralleldesc;
1436 }
1437
1438 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1439     $confdesc->{"serial$i"} = $serialdesc;
1440 }
1441
1442 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1443     $confdesc->{"hostpci$i"} = $hostpcidesc;
1444 }
1445
1446 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1447     $drivename_hash->{"ide$i"} = 1;
1448     $confdesc->{"ide$i"} = $idedesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1452     $drivename_hash->{"sata$i"} = 1;
1453     $confdesc->{"sata$i"} = $satadesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1457     $drivename_hash->{"scsi$i"} = 1;
1458     $confdesc->{"scsi$i"} = $scsidesc ;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1462     $drivename_hash->{"virtio$i"} = 1;
1463     $confdesc->{"virtio$i"} = $virtiodesc;
1464 }
1465
1466 $drivename_hash->{efidisk0} = 1;
1467 $confdesc->{efidisk0} = $efidisk_desc;
1468
1469 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1470     $confdesc->{"usb$i"} = $usbdesc;
1471 }
1472
1473 my $unuseddesc = {
1474     optional => 1,
1475     type => 'string', format => 'pve-volume-id',
1476     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1477 };
1478
1479 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1480     $confdesc->{"unused$i"} = $unuseddesc;
1481 }
1482
1483 my $kvm_api_version = 0;
1484
1485 sub kvm_version {
1486     return $kvm_api_version if $kvm_api_version;
1487
1488     open my $fh, '<', '/dev/kvm'
1489         or return undef;
1490
1491     # 0xae00 => KVM_GET_API_VERSION
1492     $kvm_api_version = ioctl($fh, 0xae00, 0);
1493
1494     return $kvm_api_version;
1495 }
1496
1497 my $kvm_user_version = {};
1498 my $kvm_mtime = {};
1499
1500 sub kvm_user_version {
1501     my ($binary) = @_;
1502
1503     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1504     my $st = stat($binary);
1505
1506     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1507     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1508         $cachedmtime == $st->mtime;
1509
1510     $kvm_user_version->{$binary} = 'unknown';
1511     $kvm_mtime->{$binary} = $st->mtime;
1512
1513     my $code = sub {
1514         my $line = shift;
1515         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1516             $kvm_user_version->{$binary} = $2;
1517         }
1518     };
1519
1520     eval { run_command([$binary, '--version'], outfunc => $code); };
1521     warn $@ if $@;
1522
1523     return $kvm_user_version->{$binary};
1524
1525 }
1526
1527 sub kernel_has_vhost_net {
1528     return -c '/dev/vhost-net';
1529 }
1530
1531 sub valid_drive_names {
1532     # order is important - used to autoselect boot disk
1533     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1534             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1535             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1536             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1537             'efidisk0');
1538 }
1539
1540 sub is_valid_drivename {
1541     my $dev = shift;
1542
1543     return defined($drivename_hash->{$dev});
1544 }
1545
1546 sub option_exists {
1547     my $key = shift;
1548     return defined($confdesc->{$key});
1549 }
1550
1551 sub nic_models {
1552     return $nic_model_list;
1553 }
1554
1555 sub os_list_description {
1556
1557     return {
1558         other => 'Other',
1559         wxp => 'Windows XP',
1560         w2k => 'Windows 2000',
1561         w2k3 =>, 'Windows 2003',
1562         w2k8 => 'Windows 2008',
1563         wvista => 'Windows Vista',
1564         win7 => 'Windows 7',
1565         win8 => 'Windows 8/2012',
1566         win10 => 'Windows 10/2016',
1567         l24 => 'Linux 2.4',
1568         l26 => 'Linux 2.6',
1569     };
1570 }
1571
1572 my $cdrom_path;
1573
1574 sub get_cdrom_path {
1575
1576     return  $cdrom_path if $cdrom_path;
1577
1578     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1579     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1580     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1581 }
1582
1583 sub get_iso_path {
1584     my ($storecfg, $vmid, $cdrom) = @_;
1585
1586     if ($cdrom eq 'cdrom') {
1587         return get_cdrom_path();
1588     } elsif ($cdrom eq 'none') {
1589         return '';
1590     } elsif ($cdrom =~ m|^/|) {
1591         return $cdrom;
1592     } else {
1593         return PVE::Storage::path($storecfg, $cdrom);
1594     }
1595 }
1596
1597 # try to convert old style file names to volume IDs
1598 sub filename_to_volume_id {
1599     my ($vmid, $file, $media) = @_;
1600
1601      if (!($file eq 'none' || $file eq 'cdrom' ||
1602           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1603
1604         return undef if $file =~ m|/|;
1605
1606         if ($media && $media eq 'cdrom') {
1607             $file = "local:iso/$file";
1608         } else {
1609             $file = "local:$vmid/$file";
1610         }
1611     }
1612
1613     return $file;
1614 }
1615
1616 sub verify_media_type {
1617     my ($opt, $vtype, $media) = @_;
1618
1619     return if !$media;
1620
1621     my $etype;
1622     if ($media eq 'disk') {
1623         $etype = 'images';
1624     } elsif ($media eq 'cdrom') {
1625         $etype = 'iso';
1626     } else {
1627         die "internal error";
1628     }
1629
1630     return if ($vtype eq $etype);
1631
1632     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1633 }
1634
1635 sub cleanup_drive_path {
1636     my ($opt, $storecfg, $drive) = @_;
1637
1638     # try to convert filesystem paths to volume IDs
1639
1640     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1641         ($drive->{file} !~ m|^/dev/.+|) &&
1642         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1643         ($drive->{file} !~ m/^\d+$/)) {
1644         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1645         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1646         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1647         verify_media_type($opt, $vtype, $drive->{media});
1648         $drive->{file} = $volid;
1649     }
1650
1651     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1652 }
1653
1654 sub parse_hotplug_features {
1655     my ($data) = @_;
1656
1657     my $res = {};
1658
1659     return $res if $data eq '0';
1660
1661     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1662
1663     foreach my $feature (PVE::Tools::split_list($data)) {
1664         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1665             $res->{$1} = 1;
1666         } else {
1667             die "invalid hotplug feature '$feature'\n";
1668         }
1669     }
1670     return $res;
1671 }
1672
1673 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1674 sub pve_verify_hotplug_features {
1675     my ($value, $noerr) = @_;
1676
1677     return $value if parse_hotplug_features($value);
1678
1679     return undef if $noerr;
1680
1681     die "unable to parse hotplug option\n";
1682 }
1683
1684 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1685 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1686 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1687 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1688 #        [,iothread=on][,serial=serial][,model=model]
1689
1690 sub parse_drive {
1691     my ($key, $data) = @_;
1692
1693     my ($interface, $index);
1694
1695     if ($key =~ m/^([^\d]+)(\d+)$/) {
1696         $interface = $1;
1697         $index = $2;
1698     } else {
1699         return undef;
1700     }
1701
1702     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1703                                      : $confdesc->{$key}->{format};
1704     if (!$desc) {
1705         warn "invalid drive key: $key\n";
1706         return undef;
1707     }
1708     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1709     return undef if !$res;
1710     $res->{interface} = $interface;
1711     $res->{index} = $index;
1712
1713     my $error = 0;
1714     foreach my $opt (qw(bps bps_rd bps_wr)) {
1715         if (my $bps = defined(delete $res->{$opt})) {
1716             if (defined($res->{"m$opt"})) {
1717                 warn "both $opt and m$opt specified\n";
1718                 ++$error;
1719                 next;
1720             }
1721             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1722         }
1723     }
1724
1725     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1726     for my $requirement (
1727         [mbps_max => 'mbps'],
1728         [mbps_rd_max => 'mbps_rd'],
1729         [mbps_wr_max => 'mbps_wr'],
1730         [miops_max => 'miops'],
1731         [miops_rd_max => 'miops_rd'],
1732         [miops_wr_max => 'miops_wr'],
1733         [bps_max_length => 'mbps_max'],
1734         [bps_rd_max_length => 'mbps_rd_max'],
1735         [bps_wr_max_length => 'mbps_wr_max'],
1736         [iops_max_length => 'iops_max'],
1737         [iops_rd_max_length => 'iops_rd_max'],
1738         [iops_wr_max_length => 'iops_wr_max']) {
1739         my ($option, $requires) = @$requirement;
1740         if ($res->{$option} && !$res->{$requires}) {
1741             warn "$option requires $requires\n";
1742             ++$error;
1743         }
1744     }
1745
1746     return undef if $error;
1747
1748     return undef if $res->{mbps_rd} && $res->{mbps};
1749     return undef if $res->{mbps_wr} && $res->{mbps};
1750     return undef if $res->{iops_rd} && $res->{iops};
1751     return undef if $res->{iops_wr} && $res->{iops};
1752
1753     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1754         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1755         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1756         return undef if $res->{interface} eq 'virtio';
1757     }
1758
1759     if (my $size = $res->{size}) {
1760         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1761     }
1762
1763     return $res;
1764 }
1765
1766 sub print_drive {
1767     my ($vmid, $drive) = @_;
1768     my $data = { %$drive };
1769     delete $data->{$_} for qw(index interface);
1770     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1771 }
1772
1773 sub scsi_inquiry {
1774     my($fh, $noerr) = @_;
1775
1776     my $SG_IO = 0x2285;
1777     my $SG_GET_VERSION_NUM = 0x2282;
1778
1779     my $versionbuf = "\x00" x 8;
1780     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1781     if (!$ret) {
1782         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1783         return undef;
1784     }
1785     my $version = unpack("I", $versionbuf);
1786     if ($version < 30000) {
1787         die "scsi generic interface too old\n"  if !$noerr;
1788         return undef;
1789     }
1790
1791     my $buf = "\x00" x 36;
1792     my $sensebuf = "\x00" x 8;
1793     my $cmd = pack("C x3 C x1", 0x12, 36);
1794
1795     # see /usr/include/scsi/sg.h
1796     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1797
1798     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1799                       length($sensebuf), 0, length($buf), $buf,
1800                       $cmd, $sensebuf, 6000);
1801
1802     $ret = ioctl($fh, $SG_IO, $packet);
1803     if (!$ret) {
1804         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1805         return undef;
1806     }
1807
1808     my @res = unpack($sg_io_hdr_t, $packet);
1809     if ($res[17] || $res[18]) {
1810         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1811         return undef;
1812     }
1813
1814     my $res = {};
1815     (my $byte0, my $byte1, $res->{vendor},
1816      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1817
1818     $res->{removable} = $byte1 & 128 ? 1 : 0;
1819     $res->{type} = $byte0 & 31;
1820
1821     return $res;
1822 }
1823
1824 sub path_is_scsi {
1825     my ($path) = @_;
1826
1827     my $fh = IO::File->new("+<$path") || return undef;
1828     my $res = scsi_inquiry($fh, 1);
1829     close($fh);
1830
1831     return $res;
1832 }
1833
1834 sub machine_type_is_q35 {
1835     my ($conf) = @_;
1836
1837     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1838 }
1839
1840 sub print_tabletdevice_full {
1841     my ($conf, $arch) = @_;
1842
1843     my $q35 = machine_type_is_q35($conf);
1844
1845     # we use uhci for old VMs because tablet driver was buggy in older qemu
1846     my $usbbus;
1847     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1848         $usbbus = 'ehci';
1849     } else {
1850         $usbbus = 'uhci';
1851     }
1852
1853     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1854 }
1855
1856 sub print_keyboarddevice_full {
1857     my ($conf, $arch, $machine) = @_;
1858
1859     return undef if $arch ne 'aarch64';
1860
1861     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1862 }
1863
1864 sub print_drivedevice_full {
1865     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1866
1867     my $device = '';
1868     my $maxdev = 0;
1869
1870     if ($drive->{interface} eq 'virtio') {
1871         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1872         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1873         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1874     } elsif ($drive->{interface} eq 'scsi') {
1875
1876         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1877         my $unit = $drive->{index} % $maxdev;
1878         my $devicetype = 'hd';
1879         my $path = '';
1880         if (drive_is_cdrom($drive)) {
1881             $devicetype = 'cd';
1882         } else {
1883             if ($drive->{file} =~ m|^/|) {
1884                 $path = $drive->{file};
1885                 if (my $info = path_is_scsi($path)) {
1886                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1887                         $devicetype = 'block';
1888                     } elsif ($info->{type} == 1) { # tape
1889                         $devicetype = 'generic';
1890                     }
1891                 }
1892             } else {
1893                  $path = PVE::Storage::path($storecfg, $drive->{file});
1894             }
1895
1896             if($path =~ m/^iscsi\:\/\//){
1897                 $devicetype = 'generic';
1898             }
1899         }
1900
1901         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1902             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1903         } else {
1904             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1905         }
1906
1907         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1908             $device .= ",rotation_rate=1";
1909         }
1910         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1911
1912     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1913         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1914         my $controller = int($drive->{index} / $maxdev);
1915         my $unit = $drive->{index} % $maxdev;
1916         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1917
1918         $device = "ide-$devicetype";
1919         if ($drive->{interface} eq 'ide') {
1920             $device .= ",bus=ide.$controller,unit=$unit";
1921         } else {
1922             $device .= ",bus=ahci$controller.$unit";
1923         }
1924         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1925
1926         if ($devicetype eq 'hd') {
1927             if (my $model = $drive->{model}) {
1928                 $model = URI::Escape::uri_unescape($model);
1929                 $device .= ",model=$model";
1930             }
1931             if ($drive->{ssd}) {
1932                 $device .= ",rotation_rate=1";
1933             }
1934         }
1935         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1936     } elsif ($drive->{interface} eq 'usb') {
1937         die "implement me";
1938         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1939     } else {
1940         die "unsupported interface type";
1941     }
1942
1943     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1944
1945     if (my $serial = $drive->{serial}) {
1946         $serial = URI::Escape::uri_unescape($serial);
1947         $device .= ",serial=$serial";
1948     }
1949
1950
1951     return $device;
1952 }
1953
1954 sub get_initiator_name {
1955     my $initiator;
1956
1957     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1958     while (defined(my $line = <$fh>)) {
1959         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1960         $initiator = $1;
1961         last;
1962     }
1963     $fh->close();
1964
1965     return $initiator;
1966 }
1967
1968 sub print_drive_full {
1969     my ($storecfg, $vmid, $drive) = @_;
1970
1971     my $path;
1972     my $volid = $drive->{file};
1973     my $format;
1974
1975     if (drive_is_cdrom($drive)) {
1976         $path = get_iso_path($storecfg, $vmid, $volid);
1977     } else {
1978         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1979         if ($storeid) {
1980             $path = PVE::Storage::path($storecfg, $volid);
1981             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1982             $format = qemu_img_format($scfg, $volname);
1983         } else {
1984             $path = $volid;
1985             $format = "raw";
1986         }
1987    }
1988
1989     my $opts = '';
1990     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1991     foreach my $o (@qemu_drive_options) {
1992         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1993     }
1994
1995     # snapshot only accepts on|off
1996     if (defined($drive->{snapshot})) {
1997         my $v = $drive->{snapshot} ? 'on' : 'off';
1998         $opts .= ",snapshot=$v";
1999     }
2000
2001     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2002         my ($dir, $qmpname) = @$type;
2003         if (my $v = $drive->{"mbps$dir"}) {
2004             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2005         }
2006         if (my $v = $drive->{"mbps${dir}_max"}) {
2007             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"bps${dir}_max_length"}) {
2010             $opts .= ",throttling.bps$qmpname-max-length=$v";
2011         }
2012         if (my $v = $drive->{"iops${dir}"}) {
2013             $opts .= ",throttling.iops$qmpname=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}_max"}) {
2016             $opts .= ",throttling.iops$qmpname-max=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max_length"}) {
2019             $opts .= ",throttling.iops$qmpname-max-length=$v";
2020         }
2021     }
2022
2023     $opts .= ",format=$format" if $format && !$drive->{format};
2024
2025     my $cache_direct = 0;
2026
2027     if (my $cache = $drive->{cache}) {
2028         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2029     } elsif (!drive_is_cdrom($drive)) {
2030         $opts .= ",cache=none";
2031         $cache_direct = 1;
2032     }
2033
2034     # aio native works only with O_DIRECT
2035     if (!$drive->{aio}) {
2036         if($cache_direct) {
2037             $opts .= ",aio=native";
2038         } else {
2039             $opts .= ",aio=threads";
2040         }
2041     }
2042
2043     if (!drive_is_cdrom($drive)) {
2044         my $detectzeroes;
2045         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2046             $detectzeroes = 'off';
2047         } elsif ($drive->{discard}) {
2048             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2049         } else {
2050             # This used to be our default with discard not being specified:
2051             $detectzeroes = 'on';
2052         }
2053         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2054     }
2055
2056     my $pathinfo = $path ? "file=$path," : '';
2057
2058     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2059 }
2060
2061 sub print_netdevice_full {
2062     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2063
2064     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2065
2066     my $device = $net->{model};
2067     if ($net->{model} eq 'virtio') {
2068          $device = 'virtio-net-pci';
2069      };
2070
2071     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2072     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2073     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2074         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2075         my $vectors = $net->{queues} * 2 + 2;
2076         $tmpstr .= ",vectors=$vectors,mq=on";
2077     }
2078     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2079
2080     if ($use_old_bios_files) {
2081         my $romfile;
2082         if ($device eq 'virtio-net-pci') {
2083             $romfile = 'pxe-virtio.rom';
2084         } elsif ($device eq 'e1000') {
2085             $romfile = 'pxe-e1000.rom';
2086         } elsif ($device eq 'ne2k') {
2087             $romfile = 'pxe-ne2k_pci.rom';
2088         } elsif ($device eq 'pcnet') {
2089             $romfile = 'pxe-pcnet.rom';
2090         } elsif ($device eq 'rtl8139') {
2091             $romfile = 'pxe-rtl8139.rom';
2092         }
2093         $tmpstr .= ",romfile=$romfile" if $romfile;
2094     }
2095
2096     return $tmpstr;
2097 }
2098
2099 sub print_netdev_full {
2100     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2101
2102     my $i = '';
2103     if ($netid =~ m/^net(\d+)$/) {
2104         $i = int($1);
2105     }
2106
2107     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2108
2109     my $ifname = "tap${vmid}i$i";
2110
2111     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2112     die "interface name '$ifname' is too long (max 15 character)\n"
2113         if length($ifname) >= 16;
2114
2115     my $vhostparam = '';
2116     if (is_native($arch)) {
2117         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2118     }
2119
2120     my $vmname = $conf->{name} || "vm$vmid";
2121
2122     my $netdev = "";
2123     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2124
2125     if ($net->{bridge}) {
2126         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2127     } else {
2128         $netdev = "type=user,id=$netid,hostname=$vmname";
2129     }
2130
2131     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2132
2133     return $netdev;
2134 }
2135
2136
2137 sub print_cpu_device {
2138     my ($conf, $id) = @_;
2139
2140     my $kvm = $conf->{kvm} // 1;
2141     my $cpu = $kvm ? "kvm64" : "qemu64";
2142     if (my $cputype = $conf->{cpu}) {
2143         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2144             or die "Cannot parse cpu description: $cputype\n";
2145         $cpu = $cpuconf->{cputype};
2146     }
2147
2148     my $cores = $conf->{cores} || 1;
2149
2150     my $current_core = ($id - 1) % $cores;
2151     my $current_socket = int(($id - 1 - $current_core)/$cores);
2152
2153     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2154 }
2155
2156 my $vga_map = {
2157     'cirrus' => 'cirrus-vga',
2158     'std' => 'VGA',
2159     'vmware' => 'vmware-svga',
2160     'virtio' => 'virtio-vga',
2161 };
2162
2163 sub print_vga_device {
2164     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2165
2166     my $type = $vga_map->{$vga->{type}};
2167     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2168         $type = 'virtio-gpu';
2169     }
2170     my $vgamem_mb = $vga->{memory};
2171     if ($qxlnum) {
2172         $type = $id ? 'qxl' : 'qxl-vga';
2173     }
2174     die "no devicetype for $vga->{type}\n" if !$type;
2175
2176     my $memory = "";
2177     if ($vgamem_mb) {
2178         if ($vga->{type} eq 'virtio') {
2179             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2180             $memory = ",max_hostmem=$bytes";
2181         } elsif ($qxlnum) {
2182             # from https://www.spice-space.org/multiple-monitors.html
2183             $memory = ",vgamem_mb=$vga->{memory}";
2184             my $ram = $vgamem_mb * 4;
2185             my $vram = $vgamem_mb * 2;
2186             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2187         } else {
2188             $memory = ",vgamem_mb=$vga->{memory}";
2189         }
2190     } elsif ($qxlnum && $id) {
2191         $memory = ",ram_size=67108864,vram_size=33554432";
2192     }
2193
2194     my $q35 = machine_type_is_q35($conf);
2195     my $vgaid = "vga" . ($id // '');
2196     my $pciaddr;
2197
2198     if ($q35 && $vgaid eq 'vga') {
2199         # the first display uses pcie.0 bus on q35 machines
2200         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2201     } else {
2202         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2203     }
2204
2205     return "$type,id=${vgaid}${memory}${pciaddr}";
2206 }
2207
2208 sub drive_is_cloudinit {
2209     my ($drive) = @_;
2210     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2211 }
2212
2213 sub drive_is_cdrom {
2214     my ($drive, $exclude_cloudinit) = @_;
2215
2216     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2217
2218     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2219
2220 }
2221
2222 sub parse_number_sets {
2223     my ($set) = @_;
2224     my $res = [];
2225     foreach my $part (split(/;/, $set)) {
2226         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2227             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2228             push @$res, [ $1, $2 ];
2229         } else {
2230             die "invalid range: $part\n";
2231         }
2232     }
2233     return $res;
2234 }
2235
2236 sub parse_numa {
2237     my ($data) = @_;
2238
2239     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2240     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2241     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2242     return $res;
2243 }
2244
2245 sub parse_hostpci {
2246     my ($value) = @_;
2247
2248     return undef if !$value;
2249
2250     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2251
2252     my @idlist = split(/;/, $res->{host});
2253     delete $res->{host};
2254     foreach my $id (@idlist) {
2255         if ($id =~ m/\./) { # full id 00:00.1
2256             push @{$res->{pciid}}, {
2257                 id => $id,
2258             };
2259         } else { # partial id 00:00
2260             $res->{pciid} = PVE::SysFSTools::lspci($id);
2261         }
2262     }
2263     return $res;
2264 }
2265
2266 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2267 sub parse_net {
2268     my ($data) = @_;
2269
2270     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2271     if ($@) {
2272         warn $@;
2273         return undef;
2274     }
2275     if (!defined($res->{macaddr})) {
2276         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2277         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2278     }
2279     return $res;
2280 }
2281
2282 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2283 sub parse_ipconfig {
2284     my ($data) = @_;
2285
2286     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2287     if ($@) {
2288         warn $@;
2289         return undef;
2290     }
2291
2292     if ($res->{gw} && !$res->{ip}) {
2293         warn 'gateway specified without specifying an IP address';
2294         return undef;
2295     }
2296     if ($res->{gw6} && !$res->{ip6}) {
2297         warn 'IPv6 gateway specified without specifying an IPv6 address';
2298         return undef;
2299     }
2300     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2301         warn 'gateway specified together with DHCP';
2302         return undef;
2303     }
2304     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2305         # gw6 + auto/dhcp
2306         warn "IPv6 gateway specified together with $res->{ip6} address";
2307         return undef;
2308     }
2309
2310     if (!$res->{ip} && !$res->{ip6}) {
2311         return { ip => 'dhcp', ip6 => 'dhcp' };
2312     }
2313
2314     return $res;
2315 }
2316
2317 sub print_net {
2318     my $net = shift;
2319
2320     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2321 }
2322
2323 sub add_random_macs {
2324     my ($settings) = @_;
2325
2326     foreach my $opt (keys %$settings) {
2327         next if $opt !~ m/^net(\d+)$/;
2328         my $net = parse_net($settings->{$opt});
2329         next if !$net;
2330         $settings->{$opt} = print_net($net);
2331     }
2332 }
2333
2334 sub vm_is_volid_owner {
2335     my ($storecfg, $vmid, $volid) = @_;
2336
2337     if ($volid !~  m|^/|) {
2338         my ($path, $owner);
2339         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2340         if ($owner && ($owner == $vmid)) {
2341             return 1;
2342         }
2343     }
2344
2345     return undef;
2346 }
2347
2348 sub split_flagged_list {
2349     my $text = shift || '';
2350     $text =~ s/[,;]/ /g;
2351     $text =~ s/^\s+//;
2352     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2353 }
2354
2355 sub join_flagged_list {
2356     my ($how, $lst) = @_;
2357     join $how, map { $lst->{$_} . $_ } keys %$lst;
2358 }
2359
2360 sub vmconfig_delete_pending_option {
2361     my ($conf, $key, $force) = @_;
2362
2363     delete $conf->{pending}->{$key};
2364     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2365     $pending_delete_hash->{$key} = $force ? '!' : '';
2366     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2367 }
2368
2369 sub vmconfig_undelete_pending_option {
2370     my ($conf, $key) = @_;
2371
2372     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2373     delete $pending_delete_hash->{$key};
2374
2375     if (%$pending_delete_hash) {
2376         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2377     } else {
2378         delete $conf->{pending}->{delete};
2379     }
2380 }
2381
2382 sub vmconfig_register_unused_drive {
2383     my ($storecfg, $vmid, $conf, $drive) = @_;
2384
2385     if (drive_is_cloudinit($drive)) {
2386         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2387         warn $@ if $@;
2388     } elsif (!drive_is_cdrom($drive)) {
2389         my $volid = $drive->{file};
2390         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2391             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2392         }
2393     }
2394 }
2395
2396 sub vmconfig_cleanup_pending {
2397     my ($conf) = @_;
2398
2399     # remove pending changes when nothing changed
2400     my $changes;
2401     foreach my $opt (keys %{$conf->{pending}}) {
2402         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2403             $changes = 1;
2404             delete $conf->{pending}->{$opt};
2405         }
2406     }
2407
2408     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2409     my $pending_delete_hash = {};
2410     while (my ($opt, $force) = each %$current_delete_hash) {
2411         if (defined($conf->{$opt})) {
2412             $pending_delete_hash->{$opt} = $force;
2413         } else {
2414             $changes = 1;
2415         }
2416     }
2417
2418     if (%$pending_delete_hash) {
2419         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2420     } else {
2421         delete $conf->{pending}->{delete};
2422     }
2423
2424     return $changes;
2425 }
2426
2427 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2428 my $smbios1_fmt = {
2429     uuid => {
2430         type => 'string',
2431         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2432         format_description => 'UUID',
2433         description => "Set SMBIOS1 UUID.",
2434         optional => 1,
2435     },
2436     version => {
2437         type => 'string',
2438         pattern => '[A-Za-z0-9+\/]+={0,2}',
2439         format_description => 'Base64 encoded string',
2440         description => "Set SMBIOS1 version.",
2441         optional => 1,
2442     },
2443     serial => {
2444         type => 'string',
2445         pattern => '[A-Za-z0-9+\/]+={0,2}',
2446         format_description => 'Base64 encoded string',
2447         description => "Set SMBIOS1 serial number.",
2448         optional => 1,
2449     },
2450     manufacturer => {
2451         type => 'string',
2452         pattern => '[A-Za-z0-9+\/]+={0,2}',
2453         format_description => 'Base64 encoded string',
2454         description => "Set SMBIOS1 manufacturer.",
2455         optional => 1,
2456     },
2457     product => {
2458         type => 'string',
2459         pattern => '[A-Za-z0-9+\/]+={0,2}',
2460         format_description => 'Base64 encoded string',
2461         description => "Set SMBIOS1 product ID.",
2462         optional => 1,
2463     },
2464     sku => {
2465         type => 'string',
2466         pattern => '[A-Za-z0-9+\/]+={0,2}',
2467         format_description => 'Base64 encoded string',
2468         description => "Set SMBIOS1 SKU string.",
2469         optional => 1,
2470     },
2471     family => {
2472         type => 'string',
2473         pattern => '[A-Za-z0-9+\/]+={0,2}',
2474         format_description => 'Base64 encoded string',
2475         description => "Set SMBIOS1 family string.",
2476         optional => 1,
2477     },
2478     base64 => {
2479         type => 'boolean',
2480         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2481         optional => 1,
2482     },
2483 };
2484
2485 sub parse_smbios1 {
2486     my ($data) = @_;
2487
2488     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2489     warn $@ if $@;
2490     return $res;
2491 }
2492
2493 sub print_smbios1 {
2494     my ($smbios1) = @_;
2495     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2496 }
2497
2498 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2499
2500 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2501 sub verify_bootdisk {
2502     my ($value, $noerr) = @_;
2503
2504     return $value if is_valid_drivename($value);
2505
2506     return undef if $noerr;
2507
2508     die "invalid boot disk '$value'\n";
2509 }
2510
2511 sub parse_watchdog {
2512     my ($value) = @_;
2513
2514     return undef if !$value;
2515
2516     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2517     warn $@ if $@;
2518     return $res;
2519 }
2520
2521 sub parse_guest_agent {
2522     my ($value) = @_;
2523
2524     return {} if !defined($value->{agent});
2525
2526     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2527     warn $@ if $@;
2528
2529     # if the agent is disabled ignore the other potentially set properties
2530     return {} if !$res->{enabled};
2531     return $res;
2532 }
2533
2534 sub parse_vga {
2535     my ($value) = @_;
2536
2537     return {} if !$value;
2538     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2539     warn $@ if $@;
2540     return $res;
2541 }
2542
2543 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2544 sub verify_usb_device {
2545     my ($value, $noerr) = @_;
2546
2547     return $value if parse_usb_device($value);
2548
2549     return undef if $noerr;
2550
2551     die "unable to parse usb device\n";
2552 }
2553
2554 # add JSON properties for create and set function
2555 sub json_config_properties {
2556     my $prop = shift;
2557
2558     foreach my $opt (keys %$confdesc) {
2559         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2560         $prop->{$opt} = $confdesc->{$opt};
2561     }
2562
2563     return $prop;
2564 }
2565
2566 # return copy of $confdesc_cloudinit to generate documentation
2567 sub cloudinit_config_properties {
2568
2569     return dclone($confdesc_cloudinit);
2570 }
2571
2572 sub check_type {
2573     my ($key, $value) = @_;
2574
2575     die "unknown setting '$key'\n" if !$confdesc->{$key};
2576
2577     my $type = $confdesc->{$key}->{type};
2578
2579     if (!defined($value)) {
2580         die "got undefined value\n";
2581     }
2582
2583     if ($value =~ m/[\n\r]/) {
2584         die "property contains a line feed\n";
2585     }
2586
2587     if ($type eq 'boolean') {
2588         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2589         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2590         die "type check ('boolean') failed - got '$value'\n";
2591     } elsif ($type eq 'integer') {
2592         return int($1) if $value =~ m/^(\d+)$/;
2593         die "type check ('integer') failed - got '$value'\n";
2594     } elsif ($type eq 'number') {
2595         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2596         die "type check ('number') failed - got '$value'\n";
2597     } elsif ($type eq 'string') {
2598         if (my $fmt = $confdesc->{$key}->{format}) {
2599             PVE::JSONSchema::check_format($fmt, $value);
2600             return $value;
2601         }
2602         $value =~ s/^\"(.*)\"$/$1/;
2603         return $value;
2604     } else {
2605         die "internal error"
2606     }
2607 }
2608
2609 sub touch_config {
2610     my ($vmid) = @_;
2611
2612     my $conf = PVE::QemuConfig->config_file($vmid);
2613     utime undef, undef, $conf;
2614 }
2615
2616 sub destroy_vm {
2617     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2618
2619     my $conffile = PVE::QemuConfig->config_file($vmid);
2620
2621     my $conf = PVE::QemuConfig->load_config($vmid);
2622
2623     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2624
2625     if ($conf->{template}) {
2626         # check if any base image is still used by a linked clone
2627         foreach_drive($conf, sub {
2628                 my ($ds, $drive) = @_;
2629
2630                 return if drive_is_cdrom($drive);
2631
2632                 my $volid = $drive->{file};
2633
2634                 return if !$volid || $volid =~ m|^/|;
2635
2636                 die "base volume '$volid' is still in use by linked cloned\n"
2637                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2638
2639         });
2640     }
2641
2642     # only remove disks owned by this VM
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         return if drive_is_cdrom($drive, 1);
2647
2648         my $volid = $drive->{file};
2649
2650         return if !$volid || $volid =~ m|^/|;
2651
2652         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2653         return if !$path || !$owner || ($owner != $vmid);
2654
2655         eval {
2656             PVE::Storage::vdisk_free($storecfg, $volid);
2657         };
2658         warn "Could not remove disk '$volid', check manually: $@" if $@;
2659
2660     });
2661
2662     if ($keep_empty_config) {
2663         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2664     } else {
2665         unlink $conffile;
2666     }
2667
2668     # also remove unused disk
2669     eval {
2670         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2671
2672         eval {
2673             PVE::Storage::foreach_volid($dl, sub {
2674                 my ($volid, $sid, $volname, $d) = @_;
2675                 PVE::Storage::vdisk_free($storecfg, $volid);
2676             });
2677         };
2678         warn $@ if $@;
2679
2680     };
2681     warn $@ if $@;
2682 }
2683
2684 sub parse_vm_config {
2685     my ($filename, $raw) = @_;
2686
2687     return undef if !defined($raw);
2688
2689     my $res = {
2690         digest => Digest::SHA::sha1_hex($raw),
2691         snapshots => {},
2692         pending => {},
2693     };
2694
2695     $filename =~ m|/qemu-server/(\d+)\.conf$|
2696         || die "got strange filename '$filename'";
2697
2698     my $vmid = $1;
2699
2700     my $conf = $res;
2701     my $descr;
2702     my $section = '';
2703
2704     my @lines = split(/\n/, $raw);
2705     foreach my $line (@lines) {
2706         next if $line =~ m/^\s*$/;
2707
2708         if ($line =~ m/^\[PENDING\]\s*$/i) {
2709             $section = 'pending';
2710             if (defined($descr)) {
2711                 $descr =~ s/\s+$//;
2712                 $conf->{description} = $descr;
2713             }
2714             $descr = undef;
2715             $conf = $res->{$section} = {};
2716             next;
2717
2718         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2719             $section = $1;
2720             if (defined($descr)) {
2721                 $descr =~ s/\s+$//;
2722                 $conf->{description} = $descr;
2723             }
2724             $descr = undef;
2725             $conf = $res->{snapshots}->{$section} = {};
2726             next;
2727         }
2728
2729         if ($line =~ m/^\#(.*)\s*$/) {
2730             $descr = '' if !defined($descr);
2731             $descr .= PVE::Tools::decode_text($1) . "\n";
2732             next;
2733         }
2734
2735         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2736             $descr = '' if !defined($descr);
2737             $descr .= PVE::Tools::decode_text($2);
2738         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2739             $conf->{snapstate} = $1;
2740         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2741             my $key = $1;
2742             my $value = $2;
2743             $conf->{$key} = $value;
2744         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2745             my $value = $1;
2746             if ($section eq 'pending') {
2747                 $conf->{delete} = $value; # we parse this later
2748             } else {
2749                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2750             }
2751         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2752             my $key = $1;
2753             my $value = $2;
2754             eval { $value = check_type($key, $value); };
2755             if ($@) {
2756                 warn "vm $vmid - unable to parse value of '$key' - $@";
2757             } else {
2758                 $key = 'ide2' if $key eq 'cdrom';
2759                 my $fmt = $confdesc->{$key}->{format};
2760                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2761                     my $v = parse_drive($key, $value);
2762                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2763                         $v->{file} = $volid;
2764                         $value = print_drive($vmid, $v);
2765                     } else {
2766                         warn "vm $vmid - unable to parse value of '$key'\n";
2767                         next;
2768                     }
2769                 }
2770
2771                 $conf->{$key} = $value;
2772             }
2773         }
2774     }
2775
2776     if (defined($descr)) {
2777         $descr =~ s/\s+$//;
2778         $conf->{description} = $descr;
2779     }
2780     delete $res->{snapstate}; # just to be sure
2781
2782     return $res;
2783 }
2784
2785 sub write_vm_config {
2786     my ($filename, $conf) = @_;
2787
2788     delete $conf->{snapstate}; # just to be sure
2789
2790     if ($conf->{cdrom}) {
2791         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2792         $conf->{ide2} = $conf->{cdrom};
2793         delete $conf->{cdrom};
2794     }
2795
2796     # we do not use 'smp' any longer
2797     if ($conf->{sockets}) {
2798         delete $conf->{smp};
2799     } elsif ($conf->{smp}) {
2800         $conf->{sockets} = $conf->{smp};
2801         delete $conf->{cores};
2802         delete $conf->{smp};
2803     }
2804
2805     my $used_volids = {};
2806
2807     my $cleanup_config = sub {
2808         my ($cref, $pending, $snapname) = @_;
2809
2810         foreach my $key (keys %$cref) {
2811             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2812                 $key eq 'snapstate' || $key eq 'pending';
2813             my $value = $cref->{$key};
2814             if ($key eq 'delete') {
2815                 die "propertry 'delete' is only allowed in [PENDING]\n"
2816                     if !$pending;
2817                 # fixme: check syntax?
2818                 next;
2819             }
2820             eval { $value = check_type($key, $value); };
2821             die "unable to parse value of '$key' - $@" if $@;
2822
2823             $cref->{$key} = $value;
2824
2825             if (!$snapname && is_valid_drivename($key)) {
2826                 my $drive = parse_drive($key, $value);
2827                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2828             }
2829         }
2830     };
2831
2832     &$cleanup_config($conf);
2833
2834     &$cleanup_config($conf->{pending}, 1);
2835
2836     foreach my $snapname (keys %{$conf->{snapshots}}) {
2837         die "internal error" if $snapname eq 'pending';
2838         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2839     }
2840
2841     # remove 'unusedX' settings if we re-add a volume
2842     foreach my $key (keys %$conf) {
2843         my $value = $conf->{$key};
2844         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2845             delete $conf->{$key};
2846         }
2847     }
2848
2849     my $generate_raw_config = sub {
2850         my ($conf, $pending) = @_;
2851
2852         my $raw = '';
2853
2854         # add description as comment to top of file
2855         if (defined(my $descr = $conf->{description})) {
2856             if ($descr) {
2857                 foreach my $cl (split(/\n/, $descr)) {
2858                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2859                 }
2860             } else {
2861                 $raw .= "#\n" if $pending;
2862             }
2863         }
2864
2865         foreach my $key (sort keys %$conf) {
2866             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2867             $raw .= "$key: $conf->{$key}\n";
2868         }
2869         return $raw;
2870     };
2871
2872     my $raw = &$generate_raw_config($conf);
2873
2874     if (scalar(keys %{$conf->{pending}})){
2875         $raw .= "\n[PENDING]\n";
2876         $raw .= &$generate_raw_config($conf->{pending}, 1);
2877     }
2878
2879     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2880         $raw .= "\n[$snapname]\n";
2881         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2882     }
2883
2884     return $raw;
2885 }
2886
2887 sub load_defaults {
2888
2889     my $res = {};
2890
2891     # we use static defaults from our JSON schema configuration
2892     foreach my $key (keys %$confdesc) {
2893         if (defined(my $default = $confdesc->{$key}->{default})) {
2894             $res->{$key} = $default;
2895         }
2896     }
2897
2898     return $res;
2899 }
2900
2901 sub config_list {
2902     my $vmlist = PVE::Cluster::get_vmlist();
2903     my $res = {};
2904     return $res if !$vmlist || !$vmlist->{ids};
2905     my $ids = $vmlist->{ids};
2906
2907     foreach my $vmid (keys %$ids) {
2908         my $d = $ids->{$vmid};
2909         next if !$d->{node} || $d->{node} ne $nodename;
2910         next if !$d->{type} || $d->{type} ne 'qemu';
2911         $res->{$vmid}->{exists} = 1;
2912     }
2913     return $res;
2914 }
2915
2916 # test if VM uses local resources (to prevent migration)
2917 sub check_local_resources {
2918     my ($conf, $noerr) = @_;
2919
2920     my @loc_res = ();
2921
2922     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2923     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2924
2925     push @loc_res, "ivshmem" if $conf->{ivshmem};
2926
2927     foreach my $k (keys %$conf) {
2928         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2929         # sockets are safe: they will recreated be on the target side post-migrate
2930         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2931         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2932     }
2933
2934     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2935
2936     return \@loc_res;
2937 }
2938
2939 # check if used storages are available on all nodes (use by migrate)
2940 sub check_storage_availability {
2941     my ($storecfg, $conf, $node) = @_;
2942
2943     foreach_drive($conf, sub {
2944         my ($ds, $drive) = @_;
2945
2946         my $volid = $drive->{file};
2947         return if !$volid;
2948
2949         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2950         return if !$sid;
2951
2952         # check if storage is available on both nodes
2953         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2954         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2955    });
2956 }
2957
2958 # list nodes where all VM images are available (used by has_feature API)
2959 sub shared_nodes {
2960     my ($conf, $storecfg) = @_;
2961
2962     my $nodelist = PVE::Cluster::get_nodelist();
2963     my $nodehash = { map { $_ => 1 } @$nodelist };
2964     my $nodename = PVE::INotify::nodename();
2965
2966     foreach_drive($conf, sub {
2967         my ($ds, $drive) = @_;
2968
2969         my $volid = $drive->{file};
2970         return if !$volid;
2971
2972         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2973         if ($storeid) {
2974             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2975             if ($scfg->{disable}) {
2976                 $nodehash = {};
2977             } elsif (my $avail = $scfg->{nodes}) {
2978                 foreach my $node (keys %$nodehash) {
2979                     delete $nodehash->{$node} if !$avail->{$node};
2980                 }
2981             } elsif (!$scfg->{shared}) {
2982                 foreach my $node (keys %$nodehash) {
2983                     delete $nodehash->{$node} if $node ne $nodename
2984                 }
2985             }
2986         }
2987     });
2988
2989     return $nodehash
2990 }
2991
2992 sub check_local_storage_availability {
2993     my ($conf, $storecfg) = @_;
2994
2995     my $nodelist = PVE::Cluster::get_nodelist();
2996     my $nodehash = { map { $_ => {} } @$nodelist };
2997
2998     foreach_drive($conf, sub {
2999         my ($ds, $drive) = @_;
3000
3001         my $volid = $drive->{file};
3002         return if !$volid;
3003
3004         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3005         if ($storeid) {
3006             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3007
3008             if ($scfg->{disable}) {
3009                 foreach my $node (keys %$nodehash) {
3010                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3011                 }
3012             } elsif (my $avail = $scfg->{nodes}) {
3013                 foreach my $node (keys %$nodehash) {
3014                     if (!$avail->{$node}) {
3015                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3016                     }
3017                 }
3018             }
3019         }
3020     });
3021
3022     foreach my $node (values %$nodehash) {
3023         if (my $unavail = $node->{unavailable_storages}) {
3024             $node->{unavailable_storages} = [ sort keys %$unavail ];
3025         }
3026     }
3027
3028     return $nodehash
3029 }
3030
3031 sub check_cmdline {
3032     my ($pidfile, $pid) = @_;
3033
3034     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3035     if (defined($fh)) {
3036         my $line = <$fh>;
3037         $fh->close;
3038         return undef if !$line;
3039         my @param = split(/\0/, $line);
3040
3041         my $cmd = $param[0];
3042         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3043
3044         for (my $i = 0; $i < scalar (@param); $i++) {
3045             my $p = $param[$i];
3046             next if !$p;
3047             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3048                 my $p = $param[$i+1];
3049                 return 1 if $p && ($p eq $pidfile);
3050                 return undef;
3051             }
3052         }
3053     }
3054     return undef;
3055 }
3056
3057 sub check_running {
3058     my ($vmid, $nocheck, $node) = @_;
3059
3060     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3061
3062     die "unable to find configuration file for VM $vmid - no such machine\n"
3063         if !$nocheck && ! -f $filename;
3064
3065     my $pidfile = pidfile_name($vmid);
3066
3067     if (my $fd = IO::File->new("<$pidfile")) {
3068         my $st = stat($fd);
3069         my $line = <$fd>;
3070         close($fd);
3071
3072         my $mtime = $st->mtime;
3073         if ($mtime > time()) {
3074             warn "file '$filename' modified in future\n";
3075         }
3076
3077         if ($line =~ m/^(\d+)$/) {
3078             my $pid = $1;
3079             if (check_cmdline($pidfile, $pid)) {
3080                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3081                     return $pid;
3082                 }
3083             }
3084         }
3085     }
3086
3087     return undef;
3088 }
3089
3090 sub vzlist {
3091
3092     my $vzlist = config_list();
3093
3094     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3095
3096     while (defined(my $de = $fd->read)) {
3097         next if $de !~ m/^(\d+)\.pid$/;
3098         my $vmid = $1;
3099         next if !defined($vzlist->{$vmid});
3100         if (my $pid = check_running($vmid)) {
3101             $vzlist->{$vmid}->{pid} = $pid;
3102         }
3103     }
3104
3105     return $vzlist;
3106 }
3107
3108 sub disksize {
3109     my ($storecfg, $conf) = @_;
3110
3111     my $bootdisk = $conf->{bootdisk};
3112     return undef if !$bootdisk;
3113     return undef if !is_valid_drivename($bootdisk);
3114
3115     return undef if !$conf->{$bootdisk};
3116
3117     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3118     return undef if !defined($drive);
3119
3120     return undef if drive_is_cdrom($drive);
3121
3122     my $volid = $drive->{file};
3123     return undef if !$volid;
3124
3125     return $drive->{size};
3126 }
3127
3128 our $vmstatus_return_properties = {
3129     vmid => get_standard_option('pve-vmid'),
3130     status => {
3131         description => "Qemu process status.",
3132         type => 'string',
3133         enum => ['stopped', 'running'],
3134     },
3135     maxmem => {
3136         description => "Maximum memory in bytes.",
3137         type => 'integer',
3138         optional => 1,
3139         renderer => 'bytes',
3140     },
3141     maxdisk => {
3142         description => "Root disk size in bytes.",
3143         type => 'integer',
3144         optional => 1,
3145         renderer => 'bytes',
3146     },
3147     name => {
3148         description => "VM name.",
3149         type => 'string',
3150         optional => 1,
3151     },
3152     qmpstatus => {
3153         description => "Qemu QMP agent status.",
3154         type => 'string',
3155         optional => 1,
3156     },
3157     pid => {
3158         description => "PID of running qemu process.",
3159         type => 'integer',
3160         optional => 1,
3161     },
3162     uptime => {
3163         description => "Uptime.",
3164         type => 'integer',
3165         optional => 1,
3166         renderer => 'duration',
3167     },
3168     cpus => {
3169         description => "Maximum usable CPUs.",
3170         type => 'number',
3171         optional => 1,
3172     },
3173     lock => {
3174         description => "The current config lock, if any.",
3175         type => 'string',
3176         optional => 1,
3177     }
3178 };
3179
3180 my $last_proc_pid_stat;
3181
3182 # get VM status information
3183 # This must be fast and should not block ($full == false)
3184 # We only query KVM using QMP if $full == true (this can be slow)
3185 sub vmstatus {
3186     my ($opt_vmid, $full) = @_;
3187
3188     my $res = {};
3189
3190     my $storecfg = PVE::Storage::config();
3191
3192     my $list = vzlist();
3193     my $defaults = load_defaults();
3194
3195     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3196
3197     my $cpucount = $cpuinfo->{cpus} || 1;
3198
3199     foreach my $vmid (keys %$list) {
3200         next if $opt_vmid && ($vmid ne $opt_vmid);
3201
3202         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3203         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3204
3205         my $d = { vmid => $vmid };
3206         $d->{pid} = $list->{$vmid}->{pid};
3207
3208         # fixme: better status?
3209         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3210
3211         my $size = disksize($storecfg, $conf);
3212         if (defined($size)) {
3213             $d->{disk} = 0; # no info available
3214             $d->{maxdisk} = $size;
3215         } else {
3216             $d->{disk} = 0;
3217             $d->{maxdisk} = 0;
3218         }
3219
3220         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3221             * ($conf->{cores} || $defaults->{cores});
3222         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3223         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3224
3225         $d->{name} = $conf->{name} || "VM $vmid";
3226         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3227             : $defaults->{memory}*(1024*1024);
3228
3229         if ($conf->{balloon}) {
3230             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3231             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3232                 : $defaults->{shares};
3233         }
3234
3235         $d->{uptime} = 0;
3236         $d->{cpu} = 0;
3237         $d->{mem} = 0;
3238
3239         $d->{netout} = 0;
3240         $d->{netin} = 0;
3241
3242         $d->{diskread} = 0;
3243         $d->{diskwrite} = 0;
3244
3245         $d->{template} = PVE::QemuConfig->is_template($conf);
3246
3247         $d->{serial} = 1 if conf_has_serial($conf);
3248         $d->{lock} = $conf->{lock} if $conf->{lock};
3249
3250         $res->{$vmid} = $d;
3251     }
3252
3253     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3254     foreach my $dev (keys %$netdev) {
3255         next if $dev !~ m/^tap([1-9]\d*)i/;
3256         my $vmid = $1;
3257         my $d = $res->{$vmid};
3258         next if !$d;
3259
3260         $d->{netout} += $netdev->{$dev}->{receive};
3261         $d->{netin} += $netdev->{$dev}->{transmit};
3262
3263         if ($full) {
3264             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3265             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3266         }
3267
3268     }
3269
3270     my $ctime = gettimeofday;
3271
3272     foreach my $vmid (keys %$list) {
3273
3274         my $d = $res->{$vmid};
3275         my $pid = $d->{pid};
3276         next if !$pid;
3277
3278         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3279         next if !$pstat; # not running
3280
3281         my $used = $pstat->{utime} + $pstat->{stime};
3282
3283         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3284
3285         if ($pstat->{vsize}) {
3286             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3287         }
3288
3289         my $old = $last_proc_pid_stat->{$pid};
3290         if (!$old) {
3291             $last_proc_pid_stat->{$pid} = {
3292                 time => $ctime,
3293                 used => $used,
3294                 cpu => 0,
3295             };
3296             next;
3297         }
3298
3299         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3300
3301         if ($dtime > 1000) {
3302             my $dutime = $used -  $old->{used};
3303
3304             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3305             $last_proc_pid_stat->{$pid} = {
3306                 time => $ctime,
3307                 used => $used,
3308                 cpu => $d->{cpu},
3309             };
3310         } else {
3311             $d->{cpu} = $old->{cpu};
3312         }
3313     }
3314
3315     return $res if !$full;
3316
3317     my $qmpclient = PVE::QMPClient->new();
3318
3319     my $ballooncb = sub {
3320         my ($vmid, $resp) = @_;
3321
3322         my $info = $resp->{'return'};
3323         return if !$info->{max_mem};
3324
3325         my $d = $res->{$vmid};
3326
3327         # use memory assigned to VM
3328         $d->{maxmem} = $info->{max_mem};
3329         $d->{balloon} = $info->{actual};
3330
3331         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3332             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3333             $d->{freemem} = $info->{free_mem};
3334         }
3335
3336         $d->{ballooninfo} = $info;
3337     };
3338
3339     my $blockstatscb = sub {
3340         my ($vmid, $resp) = @_;
3341         my $data = $resp->{'return'} || [];
3342         my $totalrdbytes = 0;
3343         my $totalwrbytes = 0;
3344
3345         for my $blockstat (@$data) {
3346             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3347             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3348
3349             $blockstat->{device} =~ s/drive-//;
3350             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3351         }
3352         $res->{$vmid}->{diskread} = $totalrdbytes;
3353         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3354     };
3355
3356     my $statuscb = sub {
3357         my ($vmid, $resp) = @_;
3358
3359         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3360         # this fails if ballon driver is not loaded, so this must be
3361         # the last commnand (following command are aborted if this fails).
3362         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3363
3364         my $status = 'unknown';
3365         if (!defined($status = $resp->{'return'}->{status})) {
3366             warn "unable to get VM status\n";
3367             return;
3368         }
3369
3370         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3371     };
3372
3373     foreach my $vmid (keys %$list) {
3374         next if $opt_vmid && ($vmid ne $opt_vmid);
3375         next if !$res->{$vmid}->{pid}; # not running
3376         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3377     }
3378
3379     $qmpclient->queue_execute(undef, 2);
3380
3381     foreach my $vmid (keys %$list) {
3382         next if $opt_vmid && ($vmid ne $opt_vmid);
3383         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3384     }
3385
3386     return $res;
3387 }
3388
3389 sub foreach_drive {
3390     my ($conf, $func, @param) = @_;
3391
3392     foreach my $ds (valid_drive_names()) {
3393         next if !defined($conf->{$ds});
3394
3395         my $drive = parse_drive($ds, $conf->{$ds});
3396         next if !$drive;
3397
3398         &$func($ds, $drive, @param);
3399     }
3400 }
3401
3402 sub foreach_volid {
3403     my ($conf, $func, @param) = @_;
3404
3405     my $volhash = {};
3406
3407     my $test_volid = sub {
3408         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3409
3410         return if !$volid;
3411
3412         $volhash->{$volid}->{cdrom} //= 1;
3413         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3414
3415         $volhash->{$volid}->{replicate} //= 0;
3416         $volhash->{$volid}->{replicate} = 1 if $replicate;
3417
3418         $volhash->{$volid}->{shared} //= 0;
3419         $volhash->{$volid}->{shared} = 1 if $shared;
3420
3421         $volhash->{$volid}->{referenced_in_config} //= 0;
3422         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3423
3424         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3425             if defined($snapname);
3426         $volhash->{$volid}->{size} = $size if $size;
3427     };
3428
3429     foreach_drive($conf, sub {
3430         my ($ds, $drive) = @_;
3431         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3432     });
3433
3434     foreach my $snapname (keys %{$conf->{snapshots}}) {
3435         my $snap = $conf->{snapshots}->{$snapname};
3436         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3437         foreach_drive($snap, sub {
3438             my ($ds, $drive) = @_;
3439             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3440         });
3441     }
3442
3443     foreach my $volid (keys %$volhash) {
3444         &$func($volid, $volhash->{$volid}, @param);
3445     }
3446 }
3447
3448 sub conf_has_serial {
3449     my ($conf) = @_;
3450
3451     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3452         if ($conf->{"serial$i"}) {
3453             return 1;
3454         }
3455     }
3456
3457     return 0;
3458 }
3459
3460 sub conf_has_audio {
3461     my ($conf, $id) = @_;
3462
3463     $id //= 0;
3464     my $audio = $conf->{"audio$id"};
3465     return undef if !defined($audio);
3466
3467     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3468     my $audiodriver = $audioproperties->{driver} // 'spice';
3469
3470     return {
3471         dev => $audioproperties->{device},
3472         dev_id => "audiodev$id",
3473         backend => $audiodriver,
3474         backend_id => "$audiodriver-backend${id}",
3475     };
3476 }
3477
3478 sub vga_conf_has_spice {
3479     my ($vga) = @_;
3480
3481     my $vgaconf = parse_vga($vga);
3482     my $vgatype = $vgaconf->{type};
3483     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3484
3485     return $1 || 1;
3486 }
3487
3488 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3489 sub get_host_arch() {
3490     $host_arch = (POSIX::uname())[4] if !$host_arch;
3491     return $host_arch;
3492 }
3493
3494 sub is_native($) {
3495     my ($arch) = @_;
3496     return get_host_arch() eq $arch;
3497 }
3498
3499 my $default_machines = {
3500     x86_64 => 'pc',
3501     aarch64 => 'virt',
3502 };
3503
3504 sub get_basic_machine_info {
3505     my ($conf, $forcemachine) = @_;
3506
3507     my $arch = $conf->{arch} // get_host_arch();
3508     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3509     return ($arch, $machine);
3510 }
3511
3512 sub get_ovmf_files($) {
3513     my ($arch) = @_;
3514
3515     my $ovmf = $OVMF->{$arch}
3516         or die "no OVMF images known for architecture '$arch'\n";
3517
3518     return @$ovmf;
3519 }
3520
3521 my $Arch2Qemu = {
3522     aarch64 => '/usr/bin/qemu-system-aarch64',
3523     x86_64 => '/usr/bin/qemu-system-x86_64',
3524 };
3525 sub get_command_for_arch($) {
3526     my ($arch) = @_;
3527     return '/usr/bin/kvm' if is_native($arch);
3528
3529     my $cmd = $Arch2Qemu->{$arch}
3530         or die "don't know how to emulate architecture '$arch'\n";
3531     return $cmd;
3532 }
3533
3534 sub get_cpu_options {
3535     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3536
3537     my $cpuFlags = [];
3538     my $ostype = $conf->{ostype};
3539
3540     my $cpu = $kvm ? "kvm64" : "qemu64";
3541     if ($arch eq 'aarch64') {
3542         $cpu = 'cortex-a57';
3543     }
3544     my $hv_vendor_id;
3545     if (my $cputype = $conf->{cpu}) {
3546         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3547             or die "Cannot parse cpu description: $cputype\n";
3548         $cpu = $cpuconf->{cputype};
3549         $kvm_off = 1 if $cpuconf->{hidden};
3550         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3551
3552         if (defined(my $flags = $cpuconf->{flags})) {
3553             push @$cpuFlags, split(";", $flags);
3554         }
3555     }
3556
3557     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3558
3559     push @$cpuFlags , '-x2apic'
3560         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3561
3562     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3563
3564     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3565
3566     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3567
3568         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3569         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3570     }
3571
3572     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3573
3574     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3575
3576     push @$cpuFlags, 'kvm=off' if $kvm_off;
3577
3578     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3579         push @$cpuFlags, "vendor=${cpu_vendor}"
3580             if $cpu_vendor ne 'default';
3581     } elsif ($arch ne 'aarch64') {
3582         die "internal error"; # should not happen
3583     }
3584
3585     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3586
3587     return ('-cpu', $cpu);
3588 }
3589
3590 sub config_to_command {
3591     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3592
3593     my $cmd = [];
3594     my $globalFlags = [];
3595     my $machineFlags = [];
3596     my $rtcFlags = [];
3597     my $devices = [];
3598     my $pciaddr = '';
3599     my $bridges = {};
3600     my $vernum = 0; # unknown
3601     my $ostype = $conf->{ostype};
3602     my $winversion = windows_version($ostype);
3603     my $kvm = $conf->{kvm};
3604
3605     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3606     my $kvm_binary = get_command_for_arch($arch);
3607     my $kvmver = kvm_user_version($kvm_binary);
3608     $kvm //= 1 if is_native($arch);
3609
3610     if ($kvm) {
3611         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3612             if !defined kvm_version();
3613     }
3614
3615     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3616         $vernum = $1*1000000+$2*1000;
3617     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3618         $vernum = $1*1000000+$2*1000+$3;
3619     }
3620
3621     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3622
3623     my $have_ovz = -f '/proc/vz/vestat';
3624
3625     my $q35 = machine_type_is_q35($conf);
3626     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3627     my $use_old_bios_files = undef;
3628     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3629
3630     my $cpuunits = defined($conf->{cpuunits}) ?
3631             $conf->{cpuunits} : $defaults->{cpuunits};
3632
3633     push @$cmd, $kvm_binary;
3634
3635     push @$cmd, '-id', $vmid;
3636
3637     my $vmname = $conf->{name} || "vm$vmid";
3638
3639     push @$cmd, '-name', $vmname;
3640
3641     my $use_virtio = 0;
3642
3643     my $qmpsocket = qmp_socket($vmid);
3644     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3645     push @$cmd, '-mon', "chardev=qmp,mode=control";
3646
3647     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3648         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3649         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3650     }
3651
3652     push @$cmd, '-pidfile' , pidfile_name($vmid);
3653
3654     push @$cmd, '-daemonize';
3655
3656     if ($conf->{smbios1}) {
3657         my $smbios_conf = parse_smbios1($conf->{smbios1});
3658         if ($smbios_conf->{base64}) {
3659             # Do not pass base64 flag to qemu
3660             delete $smbios_conf->{base64};
3661             my $smbios_string = "";
3662             foreach my $key (keys %$smbios_conf) {
3663                 my $value;
3664                 if ($key eq "uuid") {
3665                     $value = $smbios_conf->{uuid}
3666                 } else {
3667                     $value = decode_base64($smbios_conf->{$key});
3668                 }
3669                 # qemu accepts any binary data, only commas need escaping by double comma
3670                 $value =~ s/,/,,/g;
3671                 $smbios_string .= "," . $key . "=" . $value if $value;
3672             }
3673             push @$cmd, '-smbios', "type=1" . $smbios_string;
3674         } else {
3675             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3676         }
3677     }
3678
3679     if ($conf->{vmgenid}) {
3680         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3681     }
3682
3683     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3684     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3685         die "uefi base image not found\n" if ! -f $ovmf_code;
3686
3687         my $path;
3688         my $format;
3689         if (my $efidisk = $conf->{efidisk0}) {
3690             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3691             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3692             $format = $d->{format};
3693             if ($storeid) {
3694                 $path = PVE::Storage::path($storecfg, $d->{file});
3695                 if (!defined($format)) {
3696                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3697                     $format = qemu_img_format($scfg, $volname);
3698                 }
3699             } else {
3700                 $path = $d->{file};
3701                 die "efidisk format must be specified\n"
3702                     if !defined($format);
3703             }
3704         } else {
3705             warn "no efidisk configured! Using temporary efivars disk.\n";
3706             $path = "/tmp/$vmid-ovmf.fd";
3707             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3708             $format = 'raw';
3709         }
3710
3711         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3712         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3713     }
3714
3715     # load q35 config
3716     if ($q35) {
3717         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3718         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3719             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3720         } else {
3721             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3722         }
3723     }
3724
3725     # add usb controllers
3726     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3727     push @$devices, @usbcontrollers if @usbcontrollers;
3728     my $vga = parse_vga($conf->{vga});
3729
3730     my $qxlnum = vga_conf_has_spice($conf->{vga});
3731     $vga->{type} = 'qxl' if $qxlnum;
3732
3733     if (!$vga->{type}) {
3734         if ($arch eq 'aarch64') {
3735             $vga->{type} = 'virtio';
3736         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3737             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3738         } else {
3739             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3740         }
3741     }
3742
3743     # enable absolute mouse coordinates (needed by vnc)
3744     my $tablet;
3745     if (defined($conf->{tablet})) {
3746         $tablet = $conf->{tablet};
3747     } else {
3748         $tablet = $defaults->{tablet};
3749         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3750         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3751     }
3752
3753     if ($tablet) {
3754         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3755         my $kbd = print_keyboarddevice_full($conf, $arch);
3756         push @$devices, '-device', $kbd if defined($kbd);
3757     }
3758
3759     my $kvm_off = 0;
3760     my $gpu_passthrough;
3761
3762     # host pci devices
3763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3764         my $id = "hostpci$i";
3765         my $d = parse_hostpci($conf->{$id});
3766         next if !$d;
3767
3768         if (my $pcie = $d->{pcie}) {
3769             die "q35 machine model is not enabled" if !$q35;
3770             # win7 wants to have the pcie devices directly on the pcie bus
3771             # instead of in the root port
3772             if ($winversion == 7) {
3773                 $pciaddr = print_pcie_addr("${id}bus0");
3774             } else {
3775                 # add more root ports if needed, 4 are present by default
3776                 # by pve-q35 cfgs, rest added here on demand.
3777                 if ($i > 3) {
3778                     push @$devices, '-device', print_pcie_root_port($i);
3779                 }
3780                 $pciaddr = print_pcie_addr($id);
3781             }
3782         } else {
3783             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3784         }
3785
3786         my $xvga = '';
3787         if ($d->{'x-vga'}) {
3788             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3789             $kvm_off = 1;
3790             $vga->{type} = 'none' if !defined($conf->{vga});
3791             $gpu_passthrough = 1;
3792         }
3793
3794         my $pcidevices = $d->{pciid};
3795         my $multifunction = 1 if @$pcidevices > 1;
3796
3797         my $sysfspath;
3798         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3799             my $pci_id = $pcidevices->[0]->{id};
3800             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3801             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3802         } elsif ($d->{mdev}) {
3803             warn "ignoring mediated device '$id' with multifunction device\n";
3804         }
3805
3806         my $j=0;
3807         foreach my $pcidevice (@$pcidevices) {
3808             my $devicestr = "vfio-pci";
3809
3810             if ($sysfspath) {
3811                 $devicestr .= ",sysfsdev=$sysfspath";
3812             } else {
3813                 $devicestr .= ",host=$pcidevice->{id}";
3814             }
3815
3816             my $mf_addr = $multifunction ? ".$j" : '';
3817             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3818
3819             if ($j == 0) {
3820                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3821                 $devicestr .= "$xvga";
3822                 $devicestr .= ",multifunction=on" if $multifunction;
3823                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3824             }
3825
3826             push @$devices, '-device', $devicestr;
3827             $j++;
3828         }
3829     }
3830
3831     # usb devices
3832     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3833     push @$devices, @usbdevices if @usbdevices;
3834     # serial devices
3835     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3836         if (my $path = $conf->{"serial$i"}) {
3837             if ($path eq 'socket') {
3838                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3839                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3840                 # On aarch64, serial0 is the UART device. Qemu only allows
3841                 # connecting UART devices via the '-serial' command line, as
3842                 # the device has a fixed slot on the hardware...
3843                 if ($arch eq 'aarch64' && $i == 0) {
3844                     push @$devices, '-serial', "chardev:serial$i";
3845                 } else {
3846                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3847                 }
3848             } else {
3849                 die "no such serial device\n" if ! -c $path;
3850                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3851                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3852             }
3853         }
3854     }
3855
3856     # parallel devices
3857     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3858         if (my $path = $conf->{"parallel$i"}) {
3859             die "no such parallel device\n" if ! -c $path;
3860             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3861             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3862             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3863         }
3864     }
3865
3866     if (my $audio = conf_has_audio($conf)) {
3867
3868         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3869
3870         my $id = $audio->{dev_id};
3871         if ($audio->{dev} eq 'AC97') {
3872             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3873         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3874             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3875             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3876             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3877         } else {
3878             die "unkown audio device '$audio->{dev}', implement me!";
3879         }
3880
3881         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3882     }
3883
3884     my $sockets = 1;
3885     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3886     $sockets = $conf->{sockets} if  $conf->{sockets};
3887
3888     my $cores = $conf->{cores} || 1;
3889
3890     my $maxcpus = $sockets * $cores;
3891
3892     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3893
3894     my $allowed_vcpus = $cpuinfo->{cpus};
3895
3896     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3897         if ($allowed_vcpus < $maxcpus);
3898
3899     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3900
3901         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3902         for (my $i = 2; $i <= $vcpus; $i++)  {
3903             my $cpustr = print_cpu_device($conf,$i);
3904             push @$cmd, '-device', $cpustr;
3905         }
3906
3907     } else {
3908
3909         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3910     }
3911     push @$cmd, '-nodefaults';
3912
3913     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3914
3915     my $bootindex_hash = {};
3916     my $i = 1;
3917     foreach my $o (split(//, $bootorder)) {
3918         $bootindex_hash->{$o} = $i*100;
3919         $i++;
3920     }
3921
3922     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3923
3924     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3925
3926     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3927
3928     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3929         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3930         my $socket = vnc_socket($vmid);
3931         push @$cmd,  '-vnc', "unix:$socket,password";
3932     } else {
3933         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3934         push @$cmd, '-nographic';
3935     }
3936
3937     # time drift fix
3938     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3939
3940     my $useLocaltime = $conf->{localtime};
3941
3942     if ($winversion >= 5) { # windows
3943         $useLocaltime = 1 if !defined($conf->{localtime});
3944
3945         # use time drift fix when acpi is enabled
3946         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3947             $tdf = 1 if !defined($conf->{tdf});
3948         }
3949     }
3950
3951     if ($winversion >= 6) {
3952         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3953         push @$cmd, '-no-hpet';
3954     }
3955
3956     push @$rtcFlags, 'driftfix=slew' if $tdf;
3957
3958     if (!$kvm) {
3959         push @$machineFlags, 'accel=tcg';
3960     }
3961
3962     if ($machine_type) {
3963         push @$machineFlags, "type=${machine_type}";
3964     }
3965
3966     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3967         push @$rtcFlags, "base=$conf->{startdate}";
3968     } elsif ($useLocaltime) {
3969         push @$rtcFlags, 'base=localtime';
3970     }
3971
3972     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3973
3974     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3975
3976     push @$cmd, '-S' if $conf->{freeze};
3977
3978     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3979
3980     if (parse_guest_agent($conf)->{enabled}) {
3981         my $qgasocket = qmp_socket($vmid, 1);
3982         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3983         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3984         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3985         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3986     }
3987
3988     my $spice_port;
3989
3990     if ($qxlnum) {
3991         if ($qxlnum > 1) {
3992             if ($winversion){
3993                 for(my $i = 1; $i < $qxlnum; $i++){
3994                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3995                 }
3996             } else {
3997                 # assume other OS works like Linux
3998                 my ($ram, $vram) = ("134217728", "67108864");
3999                 if ($vga->{memory}) {
4000                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4001                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4002                 }
4003                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4004                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4005             }
4006         }
4007
4008         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4009
4010         my $nodename = PVE::INotify::nodename();
4011         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4012         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4013         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4014         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4015         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4016
4017         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4018         if ($spice_enhancement->{foldersharing}) {
4019             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4020             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4021         }
4022
4023         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4024         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4025         push @$devices, '-spice', "$spice_opts";
4026
4027         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4028         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4029         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4030
4031     }
4032
4033     # enable balloon by default, unless explicitly disabled
4034     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4035         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4036         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4037     }
4038
4039     if ($conf->{watchdog}) {
4040         my $wdopts = parse_watchdog($conf->{watchdog});
4041         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4042         my $watchdog = $wdopts->{model} || 'i6300esb';
4043         push @$devices, '-device', "$watchdog$pciaddr";
4044         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4045     }
4046
4047     my $vollist = [];
4048     my $scsicontroller = {};
4049     my $ahcicontroller = {};
4050     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4051
4052     # Add iscsi initiator name if available
4053     if (my $initiator = get_initiator_name()) {
4054         push @$devices, '-iscsi', "initiator-name=$initiator";
4055     }
4056
4057     foreach_drive($conf, sub {
4058         my ($ds, $drive) = @_;
4059
4060         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4061             push @$vollist, $drive->{file};
4062         }
4063
4064         # ignore efidisk here, already added in bios/fw handling code above
4065         return if $drive->{interface} eq 'efidisk';
4066
4067         $use_virtio = 1 if $ds =~ m/^virtio/;
4068
4069         if (drive_is_cdrom ($drive)) {
4070             if ($bootindex_hash->{d}) {
4071                 $drive->{bootindex} = $bootindex_hash->{d};
4072                 $bootindex_hash->{d} += 1;
4073             }
4074         } else {
4075             if ($bootindex_hash->{c}) {
4076                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4077                 $bootindex_hash->{c} += 1;
4078             }
4079         }
4080
4081         if($drive->{interface} eq 'virtio'){
4082            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4083         }
4084
4085         if ($drive->{interface} eq 'scsi') {
4086
4087             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4088
4089             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4090             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4091
4092             my $iothread = '';
4093             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4094                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4095                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4096             } elsif ($drive->{iothread}) {
4097                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4098             }
4099
4100             my $queues = '';
4101             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4102                 $queues = ",num_queues=$drive->{queues}";
4103             }
4104
4105             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4106             $scsicontroller->{$controller}=1;
4107         }
4108
4109         if ($drive->{interface} eq 'sata') {
4110            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4111            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4112            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4113            $ahcicontroller->{$controller}=1;
4114         }
4115
4116         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4117         push @$devices, '-drive',$drive_cmd;
4118         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4119     });
4120
4121     for (my $i = 0; $i < $MAX_NETS; $i++) {
4122          next if !$conf->{"net$i"};
4123          my $d = parse_net($conf->{"net$i"});
4124          next if !$d;
4125
4126          $use_virtio = 1 if $d->{model} eq 'virtio';
4127
4128          if ($bootindex_hash->{n}) {
4129             $d->{bootindex} = $bootindex_hash->{n};
4130             $bootindex_hash->{n} += 1;
4131          }
4132
4133          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4134          push @$devices, '-netdev', $netdevfull;
4135
4136          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4137          push @$devices, '-device', $netdevicefull;
4138     }
4139
4140     if ($conf->{ivshmem}) {
4141         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4142
4143         my $bus;
4144         if ($q35) {
4145             $bus = print_pcie_addr("ivshmem");
4146         } else {
4147             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4148         }
4149
4150         my $ivshmem_name = $ivshmem->{name} // $vmid;
4151         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4152
4153         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4154         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4155     }
4156
4157     if (!$q35) {
4158         # add pci bridges
4159         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4160            $bridges->{1} = 1;
4161            $bridges->{2} = 1;
4162         }
4163
4164         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4165
4166         while (my ($k, $v) = each %$bridges) {
4167             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4168             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4169         }
4170     }
4171
4172     push @$cmd, @$devices;
4173     push @$cmd, '-rtc', join(',', @$rtcFlags)
4174         if scalar(@$rtcFlags);
4175     push @$cmd, '-machine', join(',', @$machineFlags)
4176         if scalar(@$machineFlags);
4177     push @$cmd, '-global', join(',', @$globalFlags)
4178         if scalar(@$globalFlags);
4179
4180     if (my $vmstate = $conf->{vmstate}) {
4181         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4182         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4183         push @$cmd, '-loadstate', $statepath;
4184     }
4185
4186     # add custom args
4187     if ($conf->{args}) {
4188         my $aa = PVE::Tools::split_args($conf->{args});
4189         push @$cmd, @$aa;
4190     }
4191
4192     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4193 }
4194
4195 sub vnc_socket {
4196     my ($vmid) = @_;
4197     return "${var_run_tmpdir}/$vmid.vnc";
4198 }
4199
4200 sub spice_port {
4201     my ($vmid) = @_;
4202
4203     my $res = vm_mon_cmd($vmid, 'query-spice');
4204
4205     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4206 }
4207
4208 sub qmp_socket {
4209     my ($vmid, $qga, $name) = @_;
4210     my $sockettype = $qga ? 'qga' : 'qmp';
4211     my $ext = $name ? '-'.$name : '';
4212     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4213 }
4214
4215 sub pidfile_name {
4216     my ($vmid) = @_;
4217     return "${var_run_tmpdir}/$vmid.pid";
4218 }
4219
4220 sub vm_devices_list {
4221     my ($vmid) = @_;
4222
4223     my $res = vm_mon_cmd($vmid, 'query-pci');
4224     my $devices_to_check = [];
4225     my $devices = {};
4226     foreach my $pcibus (@$res) {
4227         push @$devices_to_check, @{$pcibus->{devices}},
4228     }
4229
4230     while (@$devices_to_check) {
4231         my $to_check = [];
4232         for my $d (@$devices_to_check) {
4233             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4234             next if !$d->{'pci_bridge'};
4235
4236             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4237             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4238         }
4239         $devices_to_check = $to_check;
4240     }
4241
4242     my $resblock = vm_mon_cmd($vmid, 'query-block');
4243     foreach my $block (@$resblock) {
4244         if($block->{device} =~ m/^drive-(\S+)/){
4245                 $devices->{$1} = 1;
4246         }
4247     }
4248
4249     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4250     foreach my $mice (@$resmice) {
4251         if ($mice->{name} eq 'QEMU HID Tablet') {
4252             $devices->{tablet} = 1;
4253             last;
4254         }
4255     }
4256
4257     # for usb devices there is no query-usb
4258     # but we can iterate over the entries in
4259     # qom-list path=/machine/peripheral
4260     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4261     foreach my $per (@$resperipheral) {
4262         if ($per->{name} =~ m/^usb\d+$/) {
4263             $devices->{$per->{name}} = 1;
4264         }
4265     }
4266
4267     return $devices;
4268 }
4269
4270 sub vm_deviceplug {
4271     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4272
4273     my $q35 = machine_type_is_q35($conf);
4274
4275     my $devices_list = vm_devices_list($vmid);
4276     return 1 if defined($devices_list->{$deviceid});
4277
4278     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4279
4280     if ($deviceid eq 'tablet') {
4281
4282         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4283
4284     } elsif ($deviceid eq 'keyboard') {
4285
4286         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4287
4288     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4289
4290         die "usb hotplug currently not reliable\n";
4291         # since we can't reliably hot unplug all added usb devices
4292         # and usb passthrough disables live migration
4293         # we disable usb hotplugging for now
4294         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4295
4296     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4297
4298         qemu_iothread_add($vmid, $deviceid, $device);
4299
4300         qemu_driveadd($storecfg, $vmid, $device);
4301         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4302
4303         qemu_deviceadd($vmid, $devicefull);
4304         eval { qemu_deviceaddverify($vmid, $deviceid); };
4305         if (my $err = $@) {
4306             eval { qemu_drivedel($vmid, $deviceid); };
4307             warn $@ if $@;
4308             die $err;
4309         }
4310
4311     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4312
4313
4314         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4315         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4316         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4317
4318         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4319
4320         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4321             qemu_iothread_add($vmid, $deviceid, $device);
4322             $devicefull .= ",iothread=iothread-$deviceid";
4323         }
4324
4325         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4326             $devicefull .= ",num_queues=$device->{queues}";
4327         }
4328
4329         qemu_deviceadd($vmid, $devicefull);
4330         qemu_deviceaddverify($vmid, $deviceid);
4331
4332     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4333
4334         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4335         qemu_driveadd($storecfg, $vmid, $device);
4336
4337         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4338         eval { qemu_deviceadd($vmid, $devicefull); };
4339         if (my $err = $@) {
4340             eval { qemu_drivedel($vmid, $deviceid); };
4341             warn $@ if $@;
4342             die $err;
4343         }
4344
4345     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4346
4347         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4348
4349         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4350         my $use_old_bios_files = undef;
4351         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4352
4353         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4354         qemu_deviceadd($vmid, $netdevicefull);
4355         eval {
4356             qemu_deviceaddverify($vmid, $deviceid);
4357             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4358         };
4359         if (my $err = $@) {
4360             eval { qemu_netdevdel($vmid, $deviceid); };
4361             warn $@ if $@;
4362             die $err;
4363         }
4364
4365     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4366
4367         my $bridgeid = $2;
4368         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4369         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4370
4371         qemu_deviceadd($vmid, $devicefull);
4372         qemu_deviceaddverify($vmid, $deviceid);
4373
4374     } else {
4375         die "can't hotplug device '$deviceid'\n";
4376     }
4377
4378     return 1;
4379 }
4380
4381 # fixme: this should raise exceptions on error!
4382 sub vm_deviceunplug {
4383     my ($vmid, $conf, $deviceid) = @_;
4384
4385     my $devices_list = vm_devices_list($vmid);
4386     return 1 if !defined($devices_list->{$deviceid});
4387
4388     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4389
4390     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4391
4392         qemu_devicedel($vmid, $deviceid);
4393
4394     } elsif ($deviceid =~ m/^usb\d+$/) {
4395
4396         die "usb hotplug currently not reliable\n";
4397         # when unplugging usb devices this way,
4398         # there may be remaining usb controllers/hubs
4399         # so we disable it for now
4400         qemu_devicedel($vmid, $deviceid);
4401         qemu_devicedelverify($vmid, $deviceid);
4402
4403     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4404
4405         qemu_devicedel($vmid, $deviceid);
4406         qemu_devicedelverify($vmid, $deviceid);
4407         qemu_drivedel($vmid, $deviceid);
4408         qemu_iothread_del($conf, $vmid, $deviceid);
4409
4410     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4411
4412         qemu_devicedel($vmid, $deviceid);
4413         qemu_devicedelverify($vmid, $deviceid);
4414         qemu_iothread_del($conf, $vmid, $deviceid);
4415
4416     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4417
4418         qemu_devicedel($vmid, $deviceid);
4419         qemu_drivedel($vmid, $deviceid);
4420         qemu_deletescsihw($conf, $vmid, $deviceid);
4421
4422     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4423
4424         qemu_devicedel($vmid, $deviceid);
4425         qemu_devicedelverify($vmid, $deviceid);
4426         qemu_netdevdel($vmid, $deviceid);
4427
4428     } else {
4429         die "can't unplug device '$deviceid'\n";
4430     }
4431
4432     return 1;
4433 }
4434
4435 sub qemu_deviceadd {
4436     my ($vmid, $devicefull) = @_;
4437
4438     $devicefull = "driver=".$devicefull;
4439     my %options =  split(/[=,]/, $devicefull);
4440
4441     vm_mon_cmd($vmid, "device_add" , %options);
4442 }
4443
4444 sub qemu_devicedel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4448 }
4449
4450 sub qemu_iothread_add {
4451     my($vmid, $deviceid, $device) = @_;
4452
4453     if ($device->{iothread}) {
4454         my $iothreads = vm_iothreads_list($vmid);
4455         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4456     }
4457 }
4458
4459 sub qemu_iothread_del {
4460     my($conf, $vmid, $deviceid) = @_;
4461
4462     my $confid = $deviceid;
4463     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4464         $confid = 'scsi' . $1;
4465     }
4466     my $device = parse_drive($confid, $conf->{$confid});
4467     if ($device->{iothread}) {
4468         my $iothreads = vm_iothreads_list($vmid);
4469         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4470     }
4471 }
4472
4473 sub qemu_objectadd {
4474     my($vmid, $objectid, $qomtype) = @_;
4475
4476     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4477
4478     return 1;
4479 }
4480
4481 sub qemu_objectdel {
4482     my($vmid, $objectid) = @_;
4483
4484     vm_mon_cmd($vmid, "object-del", id => $objectid);
4485
4486     return 1;
4487 }
4488
4489 sub qemu_driveadd {
4490     my ($storecfg, $vmid, $device) = @_;
4491
4492     my $drive = print_drive_full($storecfg, $vmid, $device);
4493     $drive =~ s/\\/\\\\/g;
4494     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4495
4496     # If the command succeeds qemu prints: "OK"
4497     return 1 if $ret =~ m/OK/s;
4498
4499     die "adding drive failed: $ret\n";
4500 }
4501
4502 sub qemu_drivedel {
4503     my($vmid, $deviceid) = @_;
4504
4505     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4506     $ret =~ s/^\s+//;
4507
4508     return 1 if $ret eq "";
4509
4510     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4511     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4512
4513     die "deleting drive $deviceid failed : $ret\n";
4514 }
4515
4516 sub qemu_deviceaddverify {
4517     my ($vmid, $deviceid) = @_;
4518
4519     for (my $i = 0; $i <= 5; $i++) {
4520          my $devices_list = vm_devices_list($vmid);
4521          return 1 if defined($devices_list->{$deviceid});
4522          sleep 1;
4523     }
4524
4525     die "error on hotplug device '$deviceid'\n";
4526 }
4527
4528
4529 sub qemu_devicedelverify {
4530     my ($vmid, $deviceid) = @_;
4531
4532     # need to verify that the device is correctly removed as device_del
4533     # is async and empty return is not reliable
4534
4535     for (my $i = 0; $i <= 5; $i++) {
4536          my $devices_list = vm_devices_list($vmid);
4537          return 1 if !defined($devices_list->{$deviceid});
4538          sleep 1;
4539     }
4540
4541     die "error on hot-unplugging device '$deviceid'\n";
4542 }
4543
4544 sub qemu_findorcreatescsihw {
4545     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4546
4547     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4548
4549     my $scsihwid="$controller_prefix$controller";
4550     my $devices_list = vm_devices_list($vmid);
4551
4552     if(!defined($devices_list->{$scsihwid})) {
4553         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4554     }
4555
4556     return 1;
4557 }
4558
4559 sub qemu_deletescsihw {
4560     my ($conf, $vmid, $opt) = @_;
4561
4562     my $device = parse_drive($opt, $conf->{$opt});
4563
4564     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4565         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4566         return 1;
4567     }
4568
4569     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4570
4571     my $devices_list = vm_devices_list($vmid);
4572     foreach my $opt (keys %{$devices_list}) {
4573         if (PVE::QemuServer::is_valid_drivename($opt)) {
4574             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4575             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4576                 return 1;
4577             }
4578         }
4579     }
4580
4581     my $scsihwid="scsihw$controller";
4582
4583     vm_deviceunplug($vmid, $conf, $scsihwid);
4584
4585     return 1;
4586 }
4587
4588 sub qemu_add_pci_bridge {
4589     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4590
4591     my $bridges = {};
4592
4593     my $bridgeid;
4594
4595     print_pci_addr($device, $bridges, $arch, $machine_type);
4596
4597     while (my ($k, $v) = each %$bridges) {
4598         $bridgeid = $k;
4599     }
4600     return 1 if !defined($bridgeid) || $bridgeid < 1;
4601
4602     my $bridge = "pci.$bridgeid";
4603     my $devices_list = vm_devices_list($vmid);
4604
4605     if (!defined($devices_list->{$bridge})) {
4606         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4607     }
4608
4609     return 1;
4610 }
4611
4612 sub qemu_set_link_status {
4613     my ($vmid, $device, $up) = @_;
4614
4615     vm_mon_cmd($vmid, "set_link", name => $device,
4616                up => $up ? JSON::true : JSON::false);
4617 }
4618
4619 sub qemu_netdevadd {
4620     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4621
4622     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4623     my %options =  split(/[=,]/, $netdev);
4624
4625     vm_mon_cmd($vmid, "netdev_add",  %options);
4626     return 1;
4627 }
4628
4629 sub qemu_netdevdel {
4630     my ($vmid, $deviceid) = @_;
4631
4632     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4633 }
4634
4635 sub qemu_usb_hotplug {
4636     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4637
4638     return if !$device;
4639
4640     # remove the old one first
4641     vm_deviceunplug($vmid, $conf, $deviceid);
4642
4643     # check if xhci controller is necessary and available
4644     if ($device->{usb3}) {
4645
4646         my $devicelist = vm_devices_list($vmid);
4647
4648         if (!$devicelist->{xhci}) {
4649             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4650             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4651         }
4652     }
4653     my $d = parse_usb_device($device->{host});
4654     $d->{usb3} = $device->{usb3};
4655
4656     # add the new one
4657     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4658 }
4659
4660 sub qemu_cpu_hotplug {
4661     my ($vmid, $conf, $vcpus) = @_;
4662
4663     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4664
4665     my $sockets = 1;
4666     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4667     $sockets = $conf->{sockets} if  $conf->{sockets};
4668     my $cores = $conf->{cores} || 1;
4669     my $maxcpus = $sockets * $cores;
4670
4671     $vcpus = $maxcpus if !$vcpus;
4672
4673     die "you can't add more vcpus than maxcpus\n"
4674         if $vcpus > $maxcpus;
4675
4676     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4677
4678     if ($vcpus < $currentvcpus) {
4679
4680         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4681
4682             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4683                 qemu_devicedel($vmid, "cpu$i");
4684                 my $retry = 0;
4685                 my $currentrunningvcpus = undef;
4686                 while (1) {
4687                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4688                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4689                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4690                     $retry++;
4691                     sleep 1;
4692                 }
4693                 #update conf after each succesfull cpu unplug
4694                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4695                 PVE::QemuConfig->write_config($vmid, $conf);
4696             }
4697         } else {
4698             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4699         }
4700
4701         return;
4702     }
4703
4704     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4705     die "vcpus in running vm does not match its configuration\n"
4706         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4707
4708     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4709
4710         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4711             my $cpustr = print_cpu_device($conf, $i);
4712             qemu_deviceadd($vmid, $cpustr);
4713
4714             my $retry = 0;
4715             my $currentrunningvcpus = undef;
4716             while (1) {
4717                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4718                 last if scalar(@{$currentrunningvcpus}) == $i;
4719                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4720                 sleep 1;
4721                 $retry++;
4722             }
4723             #update conf after each succesfull cpu hotplug
4724             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4725             PVE::QemuConfig->write_config($vmid, $conf);
4726         }
4727     } else {
4728
4729         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4730             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4731         }
4732     }
4733 }
4734
4735 sub qemu_block_set_io_throttle {
4736     my ($vmid, $deviceid,
4737         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4738         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4739         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4740         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4741
4742     return if !check_running($vmid) ;
4743
4744     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4745         bps => int($bps),
4746         bps_rd => int($bps_rd),
4747         bps_wr => int($bps_wr),
4748         iops => int($iops),
4749         iops_rd => int($iops_rd),
4750         iops_wr => int($iops_wr),
4751         bps_max => int($bps_max),
4752         bps_rd_max => int($bps_rd_max),
4753         bps_wr_max => int($bps_wr_max),
4754         iops_max => int($iops_max),
4755         iops_rd_max => int($iops_rd_max),
4756         iops_wr_max => int($iops_wr_max),
4757         bps_max_length => int($bps_max_length),
4758         bps_rd_max_length => int($bps_rd_max_length),
4759         bps_wr_max_length => int($bps_wr_max_length),
4760         iops_max_length => int($iops_max_length),
4761         iops_rd_max_length => int($iops_rd_max_length),
4762         iops_wr_max_length => int($iops_wr_max_length),
4763     );
4764
4765 }
4766
4767 # old code, only used to shutdown old VM after update
4768 sub __read_avail {
4769     my ($fh, $timeout) = @_;
4770
4771     my $sel = new IO::Select;
4772     $sel->add($fh);
4773
4774     my $res = '';
4775     my $buf;
4776
4777     my @ready;
4778     while (scalar (@ready = $sel->can_read($timeout))) {
4779         my $count;
4780         if ($count = $fh->sysread($buf, 8192)) {
4781             if ($buf =~ /^(.*)\(qemu\) $/s) {
4782                 $res .= $1;
4783                 last;
4784             } else {
4785                 $res .= $buf;
4786             }
4787         } else {
4788             if (!defined($count)) {
4789                 die "$!\n";
4790             }
4791             last;
4792         }
4793     }
4794
4795     die "monitor read timeout\n" if !scalar(@ready);
4796
4797     return $res;
4798 }
4799
4800 sub qemu_block_resize {
4801     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4802
4803     my $running = check_running($vmid);
4804
4805     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4806
4807     return if !$running;
4808
4809     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4810
4811 }
4812
4813 sub qemu_volume_snapshot {
4814     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4815
4816     my $running = check_running($vmid);
4817
4818     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4819         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4820     } else {
4821         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4822     }
4823 }
4824
4825 sub qemu_volume_snapshot_delete {
4826     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4827
4828     my $running = check_running($vmid);
4829
4830     if($running) {
4831
4832         $running = undef;
4833         my $conf = PVE::QemuConfig->load_config($vmid);
4834         foreach_drive($conf, sub {
4835             my ($ds, $drive) = @_;
4836             $running = 1 if $drive->{file} eq $volid;
4837         });
4838     }
4839
4840     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4841         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4842     } else {
4843         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4844     }
4845 }
4846
4847 sub set_migration_caps {
4848     my ($vmid) = @_;
4849
4850     my $cap_ref = [];
4851
4852     my $enabled_cap = {
4853         "auto-converge" => 1,
4854         "xbzrle" => 1,
4855         "x-rdma-pin-all" => 0,
4856         "zero-blocks" => 0,
4857         "compress" => 0
4858     };
4859
4860     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4861
4862     for my $supported_capability (@$supported_capabilities) {
4863         push @$cap_ref, {
4864             capability => $supported_capability->{capability},
4865             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4866         };
4867     }
4868
4869     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4870 }
4871
4872 my $fast_plug_option = {
4873     'lock' => 1,
4874     'name' => 1,
4875     'onboot' => 1,
4876     'shares' => 1,
4877     'startup' => 1,
4878     'description' => 1,
4879     'protection' => 1,
4880     'vmstatestorage' => 1,
4881     'hookscript' => 1,
4882 };
4883
4884 # hotplug changes in [PENDING]
4885 # $selection hash can be used to only apply specified options, for
4886 # example: { cores => 1 } (only apply changed 'cores')
4887 # $errors ref is used to return error messages
4888 sub vmconfig_hotplug_pending {
4889     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4890
4891     my $defaults = load_defaults();
4892     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4893
4894     # commit values which do not have any impact on running VM first
4895     # Note: those option cannot raise errors, we we do not care about
4896     # $selection and always apply them.
4897
4898     my $add_error = sub {
4899         my ($opt, $msg) = @_;
4900         $errors->{$opt} = "hotplug problem - $msg";
4901     };
4902
4903     my $changes = 0;
4904     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4905         if ($fast_plug_option->{$opt}) {
4906             $conf->{$opt} = $conf->{pending}->{$opt};
4907             delete $conf->{pending}->{$opt};
4908             $changes = 1;
4909         }
4910     }
4911
4912     if ($changes) {
4913         PVE::QemuConfig->write_config($vmid, $conf);
4914         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4915     }
4916
4917     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4918
4919     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4920     while (my ($opt, $force) = each %$pending_delete_hash) {
4921         next if $selection && !$selection->{$opt};
4922         eval {
4923             if ($opt eq 'hotplug') {
4924                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4925             } elsif ($opt eq 'tablet') {
4926                 die "skip\n" if !$hotplug_features->{usb};
4927                 if ($defaults->{tablet}) {
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4929                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4930                         if $arch eq 'aarch64';
4931                 } else {
4932                     vm_deviceunplug($vmid, $conf, 'tablet');
4933                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4934                 }
4935             } elsif ($opt =~ m/^usb\d+/) {
4936                 die "skip\n";
4937                 # since we cannot reliably hot unplug usb devices
4938                 # we are disabling it
4939                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4940                 vm_deviceunplug($vmid, $conf, $opt);
4941             } elsif ($opt eq 'vcpus') {
4942                 die "skip\n" if !$hotplug_features->{cpu};
4943                 qemu_cpu_hotplug($vmid, $conf, undef);
4944             } elsif ($opt eq 'balloon') {
4945                 # enable balloon device is not hotpluggable
4946                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4947                 # here we reset the ballooning value to memory
4948                 my $balloon = $conf->{memory} || $defaults->{memory};
4949                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4950             } elsif ($fast_plug_option->{$opt}) {
4951                 # do nothing
4952             } elsif ($opt =~ m/^net(\d+)$/) {
4953                 die "skip\n" if !$hotplug_features->{network};
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif (is_valid_drivename($opt)) {
4956                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4957                 vm_deviceunplug($vmid, $conf, $opt);
4958                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4959             } elsif ($opt =~ m/^memory$/) {
4960                 die "skip\n" if !$hotplug_features->{memory};
4961                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4962             } elsif ($opt eq 'cpuunits') {
4963                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4964             } elsif ($opt eq 'cpulimit') {
4965                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4966             } else {
4967                 die "skip\n";
4968             }
4969         };
4970         if (my $err = $@) {
4971             &$add_error($opt, $err) if $err ne "skip\n";
4972         } else {
4973             # save new config if hotplug was successful
4974             delete $conf->{$opt};
4975             vmconfig_undelete_pending_option($conf, $opt);
4976             PVE::QemuConfig->write_config($vmid, $conf);
4977             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4978         }
4979     }
4980
4981     my $apply_pending_cloudinit;
4982     $apply_pending_cloudinit = sub {
4983         my ($key, $value) = @_;
4984         $apply_pending_cloudinit = sub {}; # once is enough
4985
4986         my @cloudinit_opts = keys %$confdesc_cloudinit;
4987         foreach my $opt (keys %{$conf->{pending}}) {
4988             next if !grep { $_ eq $opt } @cloudinit_opts;
4989             $conf->{$opt} = delete $conf->{pending}->{$opt};
4990         }
4991
4992         my $new_conf = { %$conf };
4993         $new_conf->{$key} = $value;
4994         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4995     };
4996
4997     foreach my $opt (keys %{$conf->{pending}}) {
4998         next if $selection && !$selection->{$opt};
4999         my $value = $conf->{pending}->{$opt};
5000         eval {
5001             if ($opt eq 'hotplug') {
5002                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5003             } elsif ($opt eq 'tablet') {
5004                 die "skip\n" if !$hotplug_features->{usb};
5005                 if ($value == 1) {
5006                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5008                         if $arch eq 'aarch64';
5009                 } elsif ($value == 0) {
5010                     vm_deviceunplug($vmid, $conf, 'tablet');
5011                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5012                 }
5013             } elsif ($opt =~ m/^usb\d+$/) {
5014                 die "skip\n";
5015                 # since we cannot reliably hot unplug usb devices
5016                 # we are disabling it
5017                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5018                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5019                 die "skip\n" if !$d;
5020                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5021             } elsif ($opt eq 'vcpus') {
5022                 die "skip\n" if !$hotplug_features->{cpu};
5023                 qemu_cpu_hotplug($vmid, $conf, $value);
5024             } elsif ($opt eq 'balloon') {
5025                 # enable/disable balloning device is not hotpluggable
5026                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5027                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5028                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5029
5030                 # allow manual ballooning if shares is set to zero
5031                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5032                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5033                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5034                 }
5035             } elsif ($opt =~ m/^net(\d+)$/) {
5036                 # some changes can be done without hotplug
5037                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5038                                     $vmid, $opt, $value, $arch, $machine_type);
5039             } elsif (is_valid_drivename($opt)) {
5040                 # some changes can be done without hotplug
5041                 my $drive = parse_drive($opt, $value);
5042                 if (drive_is_cloudinit($drive)) {
5043                     &$apply_pending_cloudinit($opt, $value);
5044                 }
5045                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5046                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5047             } elsif ($opt =~ m/^memory$/) { #dimms
5048                 die "skip\n" if !$hotplug_features->{memory};
5049                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5050             } elsif ($opt eq 'cpuunits') {
5051                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5052             } elsif ($opt eq 'cpulimit') {
5053                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5054                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5055             } else {
5056                 die "skip\n";  # skip non-hot-pluggable options
5057             }
5058         };
5059         if (my $err = $@) {
5060             &$add_error($opt, $err) if $err ne "skip\n";
5061         } else {
5062             # save new config if hotplug was successful
5063             $conf->{$opt} = $value;
5064             delete $conf->{pending}->{$opt};
5065             PVE::QemuConfig->write_config($vmid, $conf);
5066             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5067         }
5068     }
5069 }
5070
5071 sub try_deallocate_drive {
5072     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5073
5074     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5075         my $volid = $drive->{file};
5076         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5077             my $sid = PVE::Storage::parse_volume_id($volid);
5078             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5079
5080             # check if the disk is really unused
5081             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5082                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5083             PVE::Storage::vdisk_free($storecfg, $volid);
5084             return 1;
5085         } else {
5086             # If vm is not owner of this disk remove from config
5087             return 1;
5088         }
5089     }
5090
5091     return undef;
5092 }
5093
5094 sub vmconfig_delete_or_detach_drive {
5095     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5096
5097     my $drive = parse_drive($opt, $conf->{$opt});
5098
5099     my $rpcenv = PVE::RPCEnvironment::get();
5100     my $authuser = $rpcenv->get_user();
5101
5102     if ($force) {
5103         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5104         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5105     } else {
5106         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5107     }
5108 }
5109
5110 sub vmconfig_apply_pending {
5111     my ($vmid, $conf, $storecfg) = @_;
5112
5113     # cold plug
5114
5115     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5116     while (my ($opt, $force) = each %$pending_delete_hash) {
5117         die "internal error" if $opt =~ m/^unused/;
5118         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5119         if (!defined($conf->{$opt})) {
5120             vmconfig_undelete_pending_option($conf, $opt);
5121             PVE::QemuConfig->write_config($vmid, $conf);
5122         } elsif (is_valid_drivename($opt)) {
5123             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             delete $conf->{$opt};
5126             PVE::QemuConfig->write_config($vmid, $conf);
5127         } else {
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         }
5132     }
5133
5134     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5135
5136     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5137         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5138
5139         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5140             # skip if nothing changed
5141         } elsif (is_valid_drivename($opt)) {
5142             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5143                 if defined($conf->{$opt});
5144             $conf->{$opt} = $conf->{pending}->{$opt};
5145         } else {
5146             $conf->{$opt} = $conf->{pending}->{$opt};
5147         }
5148
5149         delete $conf->{pending}->{$opt};
5150         PVE::QemuConfig->write_config($vmid, $conf);
5151     }
5152 }
5153
5154 my $safe_num_ne = sub {
5155     my ($a, $b) = @_;
5156
5157     return 0 if !defined($a) && !defined($b);
5158     return 1 if !defined($a);
5159     return 1 if !defined($b);
5160
5161     return $a != $b;
5162 };
5163
5164 my $safe_string_ne = sub {
5165     my ($a, $b) = @_;
5166
5167     return 0 if !defined($a) && !defined($b);
5168     return 1 if !defined($a);
5169     return 1 if !defined($b);
5170
5171     return $a ne $b;
5172 };
5173
5174 sub vmconfig_update_net {
5175     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5176
5177     my $newnet = parse_net($value);
5178
5179     if ($conf->{$opt}) {
5180         my $oldnet = parse_net($conf->{$opt});
5181
5182         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5183             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5184             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5185             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5186
5187             # for non online change, we try to hot-unplug
5188             die "skip\n" if !$hotplug;
5189             vm_deviceunplug($vmid, $conf, $opt);
5190         } else {
5191
5192             die "internal error" if $opt !~ m/net(\d+)/;
5193             my $iface = "tap${vmid}i$1";
5194
5195             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5196                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5197                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5198                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5199                 PVE::Network::tap_unplug($iface);
5200                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5201             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5202                 # Rate can be applied on its own but any change above needs to
5203                 # include the rate in tap_plug since OVS resets everything.
5204                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5205             }
5206
5207             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5208                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5209             }
5210
5211             return 1;
5212         }
5213     }
5214
5215     if ($hotplug) {
5216         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5217     } else {
5218         die "skip\n";
5219     }
5220 }
5221
5222 sub vmconfig_update_disk {
5223     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5224
5225     # fixme: do we need force?
5226
5227     my $drive = parse_drive($opt, $value);
5228
5229     if ($conf->{$opt}) {
5230
5231         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5232
5233             my $media = $drive->{media} || 'disk';
5234             my $oldmedia = $old_drive->{media} || 'disk';
5235             die "unable to change media type\n" if $media ne $oldmedia;
5236
5237             if (!drive_is_cdrom($old_drive)) {
5238
5239                 if ($drive->{file} ne $old_drive->{file}) {
5240
5241                     die "skip\n" if !$hotplug;
5242
5243                     # unplug and register as unused
5244                     vm_deviceunplug($vmid, $conf, $opt);
5245                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5246
5247                 } else {
5248                     # update existing disk
5249
5250                     # skip non hotpluggable value
5251                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5252                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5253                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5254                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5255                         die "skip\n";
5256                     }
5257
5258                     # apply throttle
5259                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5260                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5261                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5262                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5263                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5264                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5265                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5266                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5267                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5268                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5269                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5270                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5271                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5272                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5273                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5274                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5275                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5276                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5277
5278                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5279                                                    ($drive->{mbps} || 0)*1024*1024,
5280                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5281                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5282                                                    $drive->{iops} || 0,
5283                                                    $drive->{iops_rd} || 0,
5284                                                    $drive->{iops_wr} || 0,
5285                                                    ($drive->{mbps_max} || 0)*1024*1024,
5286                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5287                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5288                                                    $drive->{iops_max} || 0,
5289                                                    $drive->{iops_rd_max} || 0,
5290                                                    $drive->{iops_wr_max} || 0,
5291                                                    $drive->{bps_max_length} || 1,
5292                                                    $drive->{bps_rd_max_length} || 1,
5293                                                    $drive->{bps_wr_max_length} || 1,
5294                                                    $drive->{iops_max_length} || 1,
5295                                                    $drive->{iops_rd_max_length} || 1,
5296                                                    $drive->{iops_wr_max_length} || 1);
5297
5298                     }
5299
5300                     return 1;
5301                 }
5302
5303             } else { # cdrom
5304
5305                 if ($drive->{file} eq 'none') {
5306                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5307                     if (drive_is_cloudinit($old_drive)) {
5308                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5309                     }
5310                 } else {
5311                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5312                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5313                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5314                 }
5315
5316                 return 1;
5317             }
5318         }
5319     }
5320
5321     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5322     # hotplug new disks
5323     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5324     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5325 }
5326
5327 sub vm_start {
5328     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5329         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5330
5331     PVE::QemuConfig->lock_config($vmid, sub {
5332         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5333
5334         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5335
5336         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5337
5338         PVE::QemuConfig->check_lock($conf)
5339             if !($skiplock || $is_suspended);
5340
5341         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5342
5343         # clean up leftover reboot request files
5344         eval { clear_reboot_request($vmid); };
5345         warn $@ if $@;
5346
5347         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5348             vmconfig_apply_pending($vmid, $conf, $storecfg);
5349             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5350         }
5351
5352         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5353
5354         my $defaults = load_defaults();
5355
5356         # set environment variable useful inside network script
5357         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5358
5359         my $local_volumes = {};
5360
5361         if ($targetstorage) {
5362             foreach_drive($conf, sub {
5363                 my ($ds, $drive) = @_;
5364
5365                 return if drive_is_cdrom($drive);
5366
5367                 my $volid = $drive->{file};
5368
5369                 return if !$volid;
5370
5371                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5372
5373                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5374                 return if $scfg->{shared};
5375                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5376             });
5377
5378             my $format = undef;
5379
5380             foreach my $opt (sort keys %$local_volumes) {
5381
5382                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5383                 my $drive = parse_drive($opt, $conf->{$opt});
5384
5385                 #if remote storage is specified, use default format
5386                 if ($targetstorage && $targetstorage ne "1") {
5387                     $storeid = $targetstorage;
5388                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5389                     $format = $defFormat;
5390                 } else {
5391                     #else we use same format than original
5392                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5393                     $format = qemu_img_format($scfg, $volid);
5394                 }
5395
5396                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5397                 my $newdrive = $drive;
5398                 $newdrive->{format} = $format;
5399                 $newdrive->{file} = $newvolid;
5400                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5401                 $local_volumes->{$opt} = $drivestr;
5402                 #pass drive to conf for command line
5403                 $conf->{$opt} = $drivestr;
5404             }
5405         }
5406
5407         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5408
5409         if ($is_suspended) {
5410             # enforce machine type on suspended vm to ensure HW compatibility
5411             $forcemachine = $conf->{runningmachine};
5412             print "Resuming suspended VM\n";
5413         }
5414
5415         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5416
5417         my $migrate_port = 0;
5418         my $migrate_uri;
5419         if ($statefile) {
5420             if ($statefile eq 'tcp') {
5421                 my $localip = "localhost";
5422                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5423                 my $nodename = PVE::INotify::nodename();
5424
5425                 if (!defined($migration_type)) {
5426                     if (defined($datacenterconf->{migration}->{type})) {
5427                         $migration_type = $datacenterconf->{migration}->{type};
5428                     } else {
5429                         $migration_type = 'secure';
5430                     }
5431                 }
5432
5433                 if ($migration_type eq 'insecure') {
5434                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5435                     if ($migrate_network_addr) {
5436                         $localip = $migrate_network_addr;
5437                     } else {
5438                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5439                     }
5440
5441                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5442                 }
5443
5444                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5445                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5446                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5447                 push @$cmd, '-incoming', $migrate_uri;
5448                 push @$cmd, '-S';
5449
5450             } elsif ($statefile eq 'unix') {
5451                 # should be default for secure migrations as a ssh TCP forward
5452                 # tunnel is not deterministic reliable ready and fails regurarly
5453                 # to set up in time, so use UNIX socket forwards
5454                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5455                 unlink $socket_addr;
5456
5457                 $migrate_uri = "unix:$socket_addr";
5458
5459                 push @$cmd, '-incoming', $migrate_uri;
5460                 push @$cmd, '-S';
5461
5462             } else {
5463                 push @$cmd, '-loadstate', $statefile;
5464             }
5465         } elsif ($paused) {
5466             push @$cmd, '-S';
5467         }
5468
5469         # host pci devices
5470         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5471           my $d = parse_hostpci($conf->{"hostpci$i"});
5472           next if !$d;
5473           my $pcidevices = $d->{pciid};
5474           foreach my $pcidevice (@$pcidevices) {
5475                 my $pciid = $pcidevice->{id};
5476
5477                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5478                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5479                 die "no pci device info for device '$pciid'\n" if !$info;
5480
5481                 if ($d->{mdev}) {
5482                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5483                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5484                 } else {
5485                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5486                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5487                     die "can't reset pci device '$pciid'\n"
5488                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5489                 }
5490           }
5491         }
5492
5493         PVE::Storage::activate_volumes($storecfg, $vollist);
5494
5495         eval {
5496             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5497                 outfunc => sub {}, errfunc => sub {});
5498         };
5499         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5500         # timeout should be more than enough here...
5501         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5502
5503         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5504                                                   : $defaults->{cpuunits};
5505
5506         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5507         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5508
5509         my %properties = (
5510             Slice => 'qemu.slice',
5511             KillMode => 'none',
5512             CPUShares => $cpuunits
5513         );
5514
5515         if (my $cpulimit = $conf->{cpulimit}) {
5516             $properties{CPUQuota} = int($cpulimit * 100);
5517         }
5518         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5519
5520         my $run_qemu = sub {
5521             PVE::Tools::run_fork sub {
5522                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5523                 run_command($cmd, %run_params);
5524             };
5525         };
5526
5527         if ($conf->{hugepages}) {
5528
5529             my $code = sub {
5530                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5531                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5532
5533                 PVE::QemuServer::Memory::hugepages_mount();
5534                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5535
5536                 eval { $run_qemu->() };
5537                 if (my $err = $@) {
5538                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5539                     die $err;
5540                 }
5541
5542                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5543             };
5544             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5545
5546         } else {
5547             eval { $run_qemu->() };
5548         }
5549
5550         if (my $err = $@) {
5551             # deactivate volumes if start fails
5552             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5553             die "start failed: $err";
5554         }
5555
5556         print "migration listens on $migrate_uri\n" if $migrate_uri;
5557
5558         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5559             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5560             warn $@ if $@;
5561         }
5562
5563         #start nbd server for storage migration
5564         if ($targetstorage) {
5565             my $nodename = PVE::INotify::nodename();
5566             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5567             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5568             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5569             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5570
5571             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5572
5573             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5574
5575             foreach my $opt (sort keys %$local_volumes) {
5576                 my $volid = $local_volumes->{$opt};
5577                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5578                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5579                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5580             }
5581         }
5582
5583         if ($migratedfrom) {
5584             eval {
5585                 set_migration_caps($vmid);
5586             };
5587             warn $@ if $@;
5588
5589             if ($spice_port) {
5590                 print "spice listens on port $spice_port\n";
5591                 if ($spice_ticket) {
5592                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5593                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5594                 }
5595             }
5596
5597         } else {
5598             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5599                 if !$statefile && $conf->{balloon};
5600
5601             foreach my $opt (keys %$conf) {
5602                 next if $opt !~  m/^net\d+$/;
5603                 my $nicconf = parse_net($conf->{$opt});
5604                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5605             }
5606         }
5607
5608         vm_mon_cmd_nocheck($vmid, 'qom-set',
5609                     path => "machine/peripheral/balloon0",
5610                     property => "guest-stats-polling-interval",
5611                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5612
5613         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5614             print "Resumed VM, removing state\n";
5615             delete $conf->@{qw(lock vmstate runningmachine)};
5616             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5617             PVE::Storage::vdisk_free($storecfg, $vmstate);
5618             PVE::QemuConfig->write_config($vmid, $conf);
5619         }
5620
5621         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5622     });
5623 }
5624
5625 sub vm_mon_cmd {
5626     my ($vmid, $execute, %params) = @_;
5627
5628     my $cmd = { execute => $execute, arguments => \%params };
5629     vm_qmp_command($vmid, $cmd);
5630 }
5631
5632 sub vm_mon_cmd_nocheck {
5633     my ($vmid, $execute, %params) = @_;
5634
5635     my $cmd = { execute => $execute, arguments => \%params };
5636     vm_qmp_command($vmid, $cmd, 1);
5637 }
5638
5639 sub vm_qmp_command {
5640     my ($vmid, $cmd, $nocheck) = @_;
5641
5642     my $res;
5643
5644     my $timeout;
5645     if ($cmd->{arguments}) {
5646         $timeout = delete $cmd->{arguments}->{timeout};
5647     }
5648
5649     eval {
5650         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5651         my $sname = qmp_socket($vmid);
5652         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5653             my $qmpclient = PVE::QMPClient->new();
5654
5655             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5656         } else {
5657             die "unable to open monitor socket\n";
5658         }
5659     };
5660     if (my $err = $@) {
5661         syslog("err", "VM $vmid qmp command failed - $err");
5662         die $err;
5663     }
5664
5665     return $res;
5666 }
5667
5668 sub vm_human_monitor_command {
5669     my ($vmid, $cmdline) = @_;
5670
5671     my $cmd = {
5672         execute => 'human-monitor-command',
5673         arguments => { 'command-line' => $cmdline},
5674     };
5675
5676     return vm_qmp_command($vmid, $cmd);
5677 }
5678
5679 sub vm_commandline {
5680     my ($storecfg, $vmid, $snapname) = @_;
5681
5682     my $conf = PVE::QemuConfig->load_config($vmid);
5683
5684     if ($snapname) {
5685         my $snapshot = $conf->{snapshots}->{$snapname};
5686         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5687
5688         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5689
5690         $conf = $snapshot;
5691     }
5692
5693     my $defaults = load_defaults();
5694
5695     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5696
5697     return PVE::Tools::cmd2string($cmd);
5698 }
5699
5700 sub vm_reset {
5701     my ($vmid, $skiplock) = @_;
5702
5703     PVE::QemuConfig->lock_config($vmid, sub {
5704
5705         my $conf = PVE::QemuConfig->load_config($vmid);
5706
5707         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5708
5709         vm_mon_cmd($vmid, "system_reset");
5710     });
5711 }
5712
5713 sub get_vm_volumes {
5714     my ($conf) = @_;
5715
5716     my $vollist = [];
5717     foreach_volid($conf, sub {
5718         my ($volid, $attr) = @_;
5719
5720         return if $volid =~ m|^/|;
5721
5722         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5723         return if !$sid;
5724
5725         push @$vollist, $volid;
5726     });
5727
5728     return $vollist;
5729 }
5730
5731 sub vm_stop_cleanup {
5732     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5733
5734     eval {
5735
5736         if (!$keepActive) {
5737             my $vollist = get_vm_volumes($conf);
5738             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5739         }
5740
5741         foreach my $ext (qw(mon qmp pid vnc qga)) {
5742             unlink "/var/run/qemu-server/${vmid}.$ext";
5743         }
5744
5745         if ($conf->{ivshmem}) {
5746             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5747             # just delete it for now, VMs which have this already open do not
5748             # are affected, but new VMs will get a separated one. If this
5749             # becomes an issue we either add some sort of ref-counting or just
5750             # add a "don't delete on stop" flag to the ivshmem format.
5751             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5752         }
5753
5754         foreach my $key (keys %$conf) {
5755             next if $key !~ m/^hostpci(\d+)$/;
5756             my $hostpciindex = $1;
5757             my $d = parse_hostpci($conf->{$key});
5758             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5759
5760             foreach my $pci (@{$d->{pciid}}) {
5761                 my $pciid = $pci->{id};
5762                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5763             }
5764         }
5765
5766         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5767     };
5768     warn $@ if $@; # avoid errors - just warn
5769 }
5770
5771 # call only in locked context
5772 sub _do_vm_stop {
5773     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5774
5775     my $pid = check_running($vmid, $nocheck);
5776     return if !$pid;
5777
5778     my $conf;
5779     if (!$nocheck) {
5780         $conf = PVE::QemuConfig->load_config($vmid);
5781         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5782         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5783             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5784             $timeout = $opts->{down} if $opts->{down};
5785         }
5786         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5787     }
5788
5789     eval {
5790         if ($shutdown) {
5791             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5792                 vm_qmp_command($vmid, {
5793                         execute => "guest-shutdown",
5794                         arguments => { timeout => $timeout }
5795                     }, $nocheck);
5796             } else {
5797                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5798             }
5799         } else {
5800             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5801         }
5802     };
5803     my $err = $@;
5804
5805     if (!$err) {
5806         $timeout = 60 if !defined($timeout);
5807
5808         my $count = 0;
5809         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5810             $count++;
5811             sleep 1;
5812         }
5813
5814         if ($count >= $timeout) {
5815             if ($force) {
5816                 warn "VM still running - terminating now with SIGTERM\n";
5817                 kill 15, $pid;
5818             } else {
5819                 die "VM quit/powerdown failed - got timeout\n";
5820             }
5821         } else {
5822             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5823             return;
5824         }
5825     } else {
5826         if ($force) {
5827             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5828             kill 15, $pid;
5829         } else {
5830             die "VM quit/powerdown failed\n";
5831         }
5832     }
5833
5834     # wait again
5835     $timeout = 10;
5836
5837     my $count = 0;
5838     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5839         $count++;
5840         sleep 1;
5841     }
5842
5843     if ($count >= $timeout) {
5844         warn "VM still running - terminating now with SIGKILL\n";
5845         kill 9, $pid;
5846         sleep 1;
5847     }
5848
5849     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5850 }
5851
5852 # Note: use $nocheck to skip tests if VM configuration file exists.
5853 # We need that when migration VMs to other nodes (files already moved)
5854 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5855 sub vm_stop {
5856     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5857
5858     $force = 1 if !defined($force) && !$shutdown;
5859
5860     if ($migratedfrom){
5861         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5862         kill 15, $pid if $pid;
5863         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5864         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5865         return;
5866     }
5867
5868     PVE::QemuConfig->lock_config($vmid, sub {
5869         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5870    });
5871 }
5872
5873 sub vm_reboot {
5874     my ($vmid, $timeout) = @_;
5875
5876     PVE::QemuConfig->lock_config($vmid, sub {
5877
5878         # only reboot if running, as qmeventd starts it again on a stop event
5879         return if !check_running($vmid);
5880
5881         create_reboot_request($vmid);
5882
5883         my $storecfg = PVE::Storage::config();
5884         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5885
5886    });
5887 }
5888
5889 sub vm_suspend {
5890     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5891
5892     my $conf;
5893     my $path;
5894     my $storecfg;
5895     my $vmstate;
5896
5897     PVE::QemuConfig->lock_config($vmid, sub {
5898
5899         $conf = PVE::QemuConfig->load_config($vmid);
5900
5901         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5902         PVE::QemuConfig->check_lock($conf)
5903             if !($skiplock || $is_backing_up);
5904
5905         die "cannot suspend to disk during backup\n"
5906             if $is_backing_up && $includestate;
5907
5908         if ($includestate) {
5909             $conf->{lock} = 'suspending';
5910             my $date = strftime("%Y-%m-%d", localtime(time()));
5911             $storecfg = PVE::Storage::config();
5912             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5913             $path = PVE::Storage::path($storecfg, $vmstate);
5914             PVE::QemuConfig->write_config($vmid, $conf);
5915         } else {
5916             vm_mon_cmd($vmid, "stop");
5917         }
5918     });
5919
5920     if ($includestate) {
5921         # save vm state
5922         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5923
5924         eval {
5925             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5926             for(;;) {
5927                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5928                 if (!$state->{status}) {
5929                     die "savevm not active\n";
5930                 } elsif ($state->{status} eq 'active') {
5931                     sleep(1);
5932                     next;
5933                 } elsif ($state->{status} eq 'completed') {
5934                     print "State saved, quitting\n";
5935                     last;
5936                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5937                     die "query-savevm failed with error '$state->{error}'\n"
5938                 } else {
5939                     die "query-savevm returned status '$state->{status}'\n";
5940                 }
5941             }
5942         };
5943         my $err = $@;
5944
5945         PVE::QemuConfig->lock_config($vmid, sub {
5946             $conf = PVE::QemuConfig->load_config($vmid);
5947             if ($err) {
5948                 # cleanup, but leave suspending lock, to indicate something went wrong
5949                 eval {
5950                     vm_mon_cmd($vmid, "savevm-end");
5951                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5952                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5953                     delete $conf->@{qw(vmstate runningmachine)};
5954                     PVE::QemuConfig->write_config($vmid, $conf);
5955                 };
5956                 warn $@ if $@;
5957                 die $err;
5958             }
5959
5960             die "lock changed unexpectedly\n"
5961                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5962
5963             vm_qmp_command($vmid, { execute => "quit" });
5964             $conf->{lock} = 'suspended';
5965             PVE::QemuConfig->write_config($vmid, $conf);
5966         });
5967     }
5968 }
5969
5970 sub vm_resume {
5971     my ($vmid, $skiplock, $nocheck) = @_;
5972
5973     PVE::QemuConfig->lock_config($vmid, sub {
5974         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5975         my $res = $vm_mon_cmd->($vmid, 'query-status');
5976         my $resume_cmd = 'cont';
5977
5978         if ($res->{status} && $res->{status} eq 'suspended') {
5979             $resume_cmd = 'system_wakeup';
5980         }
5981
5982         if (!$nocheck) {
5983
5984             my $conf = PVE::QemuConfig->load_config($vmid);
5985
5986             PVE::QemuConfig->check_lock($conf)
5987                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5988         }
5989
5990         $vm_mon_cmd->($vmid, $resume_cmd);
5991     });
5992 }
5993
5994 sub vm_sendkey {
5995     my ($vmid, $skiplock, $key) = @_;
5996
5997     PVE::QemuConfig->lock_config($vmid, sub {
5998
5999         my $conf = PVE::QemuConfig->load_config($vmid);
6000
6001         # there is no qmp command, so we use the human monitor command
6002         my $res = vm_human_monitor_command($vmid, "sendkey $key");
6003         die $res if $res ne '';
6004     });
6005 }
6006
6007 sub vm_destroy {
6008     my ($storecfg, $vmid, $skiplock) = @_;
6009
6010     PVE::QemuConfig->lock_config($vmid, sub {
6011
6012         my $conf = PVE::QemuConfig->load_config($vmid);
6013
6014         if (!check_running($vmid)) {
6015             destroy_vm($storecfg, $vmid, undef, $skiplock);
6016         } else {
6017             die "VM $vmid is running - destroy failed\n";
6018         }
6019     });
6020 }
6021
6022 # vzdump restore implementaion
6023
6024 sub tar_archive_read_firstfile {
6025     my $archive = shift;
6026
6027     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6028
6029     # try to detect archive type first
6030     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6031         die "unable to open file '$archive'\n";
6032     my $firstfile = <$fh>;
6033     kill 15, $pid;
6034     close $fh;
6035
6036     die "ERROR: archive contaions no data\n" if !$firstfile;
6037     chomp $firstfile;
6038
6039     return $firstfile;
6040 }
6041
6042 sub tar_restore_cleanup {
6043     my ($storecfg, $statfile) = @_;
6044
6045     print STDERR "starting cleanup\n";
6046
6047     if (my $fd = IO::File->new($statfile, "r")) {
6048         while (defined(my $line = <$fd>)) {
6049             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6050                 my $volid = $2;
6051                 eval {
6052                     if ($volid =~ m|^/|) {
6053                         unlink $volid || die 'unlink failed\n';
6054                     } else {
6055                         PVE::Storage::vdisk_free($storecfg, $volid);
6056                     }
6057                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6058                 };
6059                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6060             } else {
6061                 print STDERR "unable to parse line in statfile - $line";
6062             }
6063         }
6064         $fd->close();
6065     }
6066 }
6067
6068 sub restore_archive {
6069     my ($archive, $vmid, $user, $opts) = @_;
6070
6071     my $format = $opts->{format};
6072     my $comp;
6073
6074     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6075         $format = 'tar' if !$format;
6076         $comp = 'gzip';
6077     } elsif ($archive =~ m/\.tar$/) {
6078         $format = 'tar' if !$format;
6079     } elsif ($archive =~ m/.tar.lzo$/) {
6080         $format = 'tar' if !$format;
6081         $comp = 'lzop';
6082     } elsif ($archive =~ m/\.vma$/) {
6083         $format = 'vma' if !$format;
6084     } elsif ($archive =~ m/\.vma\.gz$/) {
6085         $format = 'vma' if !$format;
6086         $comp = 'gzip';
6087     } elsif ($archive =~ m/\.vma\.lzo$/) {
6088         $format = 'vma' if !$format;
6089         $comp = 'lzop';
6090     } else {
6091         $format = 'vma' if !$format; # default
6092     }
6093
6094     # try to detect archive format
6095     if ($format eq 'tar') {
6096         return restore_tar_archive($archive, $vmid, $user, $opts);
6097     } else {
6098         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6099     }
6100 }
6101
6102 sub restore_update_config_line {
6103     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6104
6105     return if $line =~ m/^\#qmdump\#/;
6106     return if $line =~ m/^\#vzdump\#/;
6107     return if $line =~ m/^lock:/;
6108     return if $line =~ m/^unused\d+:/;
6109     return if $line =~ m/^parent:/;
6110
6111     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6112     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6113         # try to convert old 1.X settings
6114         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6115         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6116             my ($model, $macaddr) = split(/\=/, $devconfig);
6117             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6118             my $net = {
6119                 model => $model,
6120                 bridge => "vmbr$ind",
6121                 macaddr => $macaddr,
6122             };
6123             my $netstr = print_net($net);
6124
6125             print $outfd "net$cookie->{netcount}: $netstr\n";
6126             $cookie->{netcount}++;
6127         }
6128     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6129         my ($id, $netstr) = ($1, $2);
6130         my $net = parse_net($netstr);
6131         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6132         $netstr = print_net($net);
6133         print $outfd "$id: $netstr\n";
6134     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6135         my $virtdev = $1;
6136         my $value = $3;
6137         my $di = parse_drive($virtdev, $value);
6138         if (defined($di->{backup}) && !$di->{backup}) {
6139             print $outfd "#$line";
6140         } elsif ($map->{$virtdev}) {
6141             delete $di->{format}; # format can change on restore
6142             $di->{file} = $map->{$virtdev};
6143             $value = print_drive($vmid, $di);
6144             print $outfd "$virtdev: $value\n";
6145         } else {
6146             print $outfd $line;
6147         }
6148     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6149         my $vmgenid = $1;
6150         if ($vmgenid ne '0') {
6151             # always generate a new vmgenid if there was a valid one setup
6152             $vmgenid = generate_uuid();
6153         }
6154         print $outfd "vmgenid: $vmgenid\n";
6155     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6156         my ($uuid, $uuid_str);
6157         UUID::generate($uuid);
6158         UUID::unparse($uuid, $uuid_str);
6159         my $smbios1 = parse_smbios1($2);
6160         $smbios1->{uuid} = $uuid_str;
6161         print $outfd $1.print_smbios1($smbios1)."\n";
6162     } else {
6163         print $outfd $line;
6164     }
6165 }
6166
6167 sub scan_volids {
6168     my ($cfg, $vmid) = @_;
6169
6170     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6171
6172     my $volid_hash = {};
6173     foreach my $storeid (keys %$info) {
6174         foreach my $item (@{$info->{$storeid}}) {
6175             next if !($item->{volid} && $item->{size});
6176             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6177             $volid_hash->{$item->{volid}} = $item;
6178         }
6179     }
6180
6181     return $volid_hash;
6182 }
6183
6184 sub is_volume_in_use {
6185     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6186
6187     my $path = PVE::Storage::path($storecfg, $volid);
6188
6189     my $scan_config = sub {
6190         my ($cref, $snapname) = @_;
6191
6192         foreach my $key (keys %$cref) {
6193             my $value = $cref->{$key};
6194             if (is_valid_drivename($key)) {
6195                 next if $skip_drive && $key eq $skip_drive;
6196                 my $drive = parse_drive($key, $value);
6197                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6198                 return 1 if $volid eq $drive->{file};
6199                 if ($drive->{file} =~ m!^/!) {
6200                     return 1 if $drive->{file} eq $path;
6201                 } else {
6202                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6203                     next if !$storeid;
6204                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6205                     next if !$scfg;
6206                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6207                 }
6208             }
6209         }
6210
6211         return 0;
6212     };
6213
6214     return 1 if &$scan_config($conf);
6215
6216     undef $skip_drive;
6217
6218     foreach my $snapname (keys %{$conf->{snapshots}}) {
6219         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6220     }
6221
6222     return 0;
6223 }
6224
6225 sub update_disksize {
6226     my ($vmid, $conf, $volid_hash) = @_;
6227
6228     my $changes;
6229     my $prefix = "VM $vmid:";
6230
6231     # used and unused disks
6232     my $referenced = {};
6233
6234     # Note: it is allowed to define multiple storages with same path (alias), so
6235     # we need to check both 'volid' and real 'path' (two different volid can point
6236     # to the same path).
6237
6238     my $referencedpath = {};
6239
6240     # update size info
6241     foreach my $opt (keys %$conf) {
6242         if (is_valid_drivename($opt)) {
6243             my $drive = parse_drive($opt, $conf->{$opt});
6244             my $volid = $drive->{file};
6245             next if !$volid;
6246
6247             $referenced->{$volid} = 1;
6248             if ($volid_hash->{$volid} &&
6249                 (my $path = $volid_hash->{$volid}->{path})) {
6250                 $referencedpath->{$path} = 1;
6251             }
6252
6253             next if drive_is_cdrom($drive);
6254             next if !$volid_hash->{$volid};
6255
6256             $drive->{size} = $volid_hash->{$volid}->{size};
6257             my $new = print_drive($vmid, $drive);
6258             if ($new ne $conf->{$opt}) {
6259                 $changes = 1;
6260                 $conf->{$opt} = $new;
6261                 print "$prefix update disk '$opt' information.\n";
6262             }
6263         }
6264     }
6265
6266     # remove 'unusedX' entry if volume is used
6267     foreach my $opt (keys %$conf) {
6268         next if $opt !~ m/^unused\d+$/;
6269         my $volid = $conf->{$opt};
6270         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6271         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6272             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6273             $changes = 1;
6274             delete $conf->{$opt};
6275         }
6276
6277         $referenced->{$volid} = 1;
6278         $referencedpath->{$path} = 1 if $path;
6279     }
6280
6281     foreach my $volid (sort keys %$volid_hash) {
6282         next if $volid =~ m/vm-$vmid-state-/;
6283         next if $referenced->{$volid};
6284         my $path = $volid_hash->{$volid}->{path};
6285         next if !$path; # just to be sure
6286         next if $referencedpath->{$path};
6287         $changes = 1;
6288         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6289         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6290         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6291     }
6292
6293     return $changes;
6294 }
6295
6296 sub rescan {
6297     my ($vmid, $nolock, $dryrun) = @_;
6298
6299     my $cfg = PVE::Storage::config();
6300
6301     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6302     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6303     foreach my $stor (keys %{$cfg->{ids}}) {
6304         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6305     }
6306
6307     print "rescan volumes...\n";
6308     my $volid_hash = scan_volids($cfg, $vmid);
6309
6310     my $updatefn =  sub {
6311         my ($vmid) = @_;
6312
6313         my $conf = PVE::QemuConfig->load_config($vmid);
6314
6315         PVE::QemuConfig->check_lock($conf);
6316
6317         my $vm_volids = {};
6318         foreach my $volid (keys %$volid_hash) {
6319             my $info = $volid_hash->{$volid};
6320             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6321         }
6322
6323         my $changes = update_disksize($vmid, $conf, $vm_volids);
6324
6325         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6326     };
6327
6328     if (defined($vmid)) {
6329         if ($nolock) {
6330             &$updatefn($vmid);
6331         } else {
6332             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6333         }
6334     } else {
6335         my $vmlist = config_list();
6336         foreach my $vmid (keys %$vmlist) {
6337             if ($nolock) {
6338                 &$updatefn($vmid);
6339             } else {
6340                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6341             }
6342         }
6343     }
6344 }
6345
6346 sub restore_vma_archive {
6347     my ($archive, $vmid, $user, $opts, $comp) = @_;
6348
6349     my $readfrom = $archive;
6350
6351     my $cfg = PVE::Storage::config();
6352     my $commands = [];
6353     my $bwlimit = $opts->{bwlimit};
6354
6355     my $dbg_cmdstring = '';
6356     my $add_pipe = sub {
6357         my ($cmd) = @_;
6358         push @$commands, $cmd;
6359         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6360         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6361         $readfrom = '-';
6362     };
6363
6364     my $input = undef;
6365     if ($archive eq '-') {
6366         $input = '<&STDIN';
6367     } else {
6368         # If we use a backup from a PVE defined storage we also consider that
6369         # storage's rate limit:
6370         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6371         if (defined($volid)) {
6372             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6373             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6374             if ($readlimit) {
6375                 print STDERR "applying read rate limit: $readlimit\n";
6376                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6377                 $add_pipe->($cstream);
6378             }
6379         }
6380     }
6381
6382     if ($comp) {
6383         my $cmd;
6384         if ($comp eq 'gzip') {
6385             $cmd = ['zcat', $readfrom];
6386         } elsif ($comp eq 'lzop') {
6387             $cmd = ['lzop', '-d', '-c', $readfrom];
6388         } else {
6389             die "unknown compression method '$comp'\n";
6390         }
6391         $add_pipe->($cmd);
6392     }
6393
6394     my $tmpdir = "/var/tmp/vzdumptmp$$";
6395     rmtree $tmpdir;
6396
6397     # disable interrupts (always do cleanups)
6398     local $SIG{INT} =
6399         local $SIG{TERM} =
6400         local $SIG{QUIT} =
6401         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6402
6403     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6404     POSIX::mkfifo($mapfifo, 0600);
6405     my $fifofh;
6406
6407     my $openfifo = sub {
6408         open($fifofh, '>', $mapfifo) || die $!;
6409     };
6410
6411     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6412
6413     my $oldtimeout;
6414     my $timeout = 5;
6415
6416     my $devinfo = {};
6417
6418     my $rpcenv = PVE::RPCEnvironment::get();
6419
6420     my $conffile = PVE::QemuConfig->config_file($vmid);
6421     my $tmpfn = "$conffile.$$.tmp";
6422
6423     # Note: $oldconf is undef if VM does not exists
6424     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6425     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6426
6427     my %storage_limits;
6428
6429     my $print_devmap = sub {
6430         my $virtdev_hash = {};
6431
6432         my $cfgfn = "$tmpdir/qemu-server.conf";
6433
6434         # we can read the config - that is already extracted
6435         my $fh = IO::File->new($cfgfn, "r") ||
6436             "unable to read qemu-server.conf - $!\n";
6437
6438         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6439         if (-f $fwcfgfn) {
6440             my $pve_firewall_dir = '/etc/pve/firewall';
6441             mkdir $pve_firewall_dir; # make sure the dir exists
6442             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6443         }
6444
6445         while (defined(my $line = <$fh>)) {
6446             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6447                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6448                 die "archive does not contain data for drive '$virtdev'\n"
6449                     if !$devinfo->{$devname};
6450                 if (defined($opts->{storage})) {
6451                     $storeid = $opts->{storage} || 'local';
6452                 } elsif (!$storeid) {
6453                     $storeid = 'local';
6454                 }
6455                 $format = 'raw' if !$format;
6456                 $devinfo->{$devname}->{devname} = $devname;
6457                 $devinfo->{$devname}->{virtdev} = $virtdev;
6458                 $devinfo->{$devname}->{format} = $format;
6459                 $devinfo->{$devname}->{storeid} = $storeid;
6460
6461                 # check permission on storage
6462                 my $pool = $opts->{pool}; # todo: do we need that?
6463                 if ($user ne 'root@pam') {
6464                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6465                 }
6466
6467                 $storage_limits{$storeid} = $bwlimit;
6468
6469                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6470             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6471                 my $virtdev = $1;
6472                 my $drive = parse_drive($virtdev, $2);
6473                 if (drive_is_cloudinit($drive)) {
6474                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6475                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6476                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6477
6478                     my $d = {
6479                         format => $format,
6480                         storeid => $opts->{storage} // $storeid,
6481                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6482                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6483                         name => "vm-$vmid-cloudinit",
6484                         is_cloudinit => 1,
6485                     };
6486                     $virtdev_hash->{$virtdev} = $d;
6487                 }
6488             }
6489         }
6490
6491         foreach my $key (keys %storage_limits) {
6492             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6493             next if !$limit;
6494             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6495             $storage_limits{$key} = $limit * 1024;
6496         }
6497
6498         foreach my $devname (keys %$devinfo) {
6499             die "found no device mapping information for device '$devname'\n"
6500                 if !$devinfo->{$devname}->{virtdev};
6501         }
6502
6503         # create empty/temp config
6504         if ($oldconf) {
6505             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6506             foreach_drive($oldconf, sub {
6507                 my ($ds, $drive) = @_;
6508
6509                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6510
6511                 my $volid = $drive->{file};
6512                 return if !$volid || $volid =~ m|^/|;
6513
6514                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6515                 return if !$path || !$owner || ($owner != $vmid);
6516
6517                 # Note: only delete disk we want to restore
6518                 # other volumes will become unused
6519                 if ($virtdev_hash->{$ds}) {
6520                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6521                     if (my $err = $@) {
6522                         warn $err;
6523                     }
6524                 }
6525             });
6526
6527             # delete vmstate files, after the restore we have no snapshots anymore
6528             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6529                 my $snap = $oldconf->{snapshots}->{$snapname};
6530                 if ($snap->{vmstate}) {
6531                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6532                     if (my $err = $@) {
6533                         warn $err;
6534                     }
6535                 }
6536             }
6537         }
6538
6539         my $map = {};
6540         foreach my $virtdev (sort keys %$virtdev_hash) {
6541             my $d = $virtdev_hash->{$virtdev};
6542             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6543             my $storeid = $d->{storeid};
6544             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6545
6546             my $map_opts = '';
6547             if (my $limit = $storage_limits{$storeid}) {
6548                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6549             }
6550
6551             # test if requested format is supported
6552             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6553             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6554             $d->{format} = $defFormat if !$supported;
6555
6556             my $name;
6557             if ($d->{is_cloudinit}) {
6558                 $name = $d->{name};
6559                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6560             }
6561
6562             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6563             print STDERR "new volume ID is '$volid'\n";
6564             $d->{volid} = $volid;
6565
6566             PVE::Storage::activate_volumes($cfg, [$volid]);
6567
6568             my $write_zeros = 1;
6569             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6570                 $write_zeros = 0;
6571             }
6572
6573             if (!$d->{is_cloudinit}) {
6574                 my $path = PVE::Storage::path($cfg, $volid);
6575
6576                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6577
6578                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6579             }
6580             $map->{$virtdev} = $volid;
6581         }
6582
6583         $fh->seek(0, 0) || die "seek failed - $!\n";
6584
6585         my $outfd = new IO::File ($tmpfn, "w") ||
6586             die "unable to write config for VM $vmid\n";
6587
6588         my $cookie = { netcount => 0 };
6589         while (defined(my $line = <$fh>)) {
6590             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6591         }
6592
6593         $fh->close();
6594         $outfd->close();
6595     };
6596
6597     eval {
6598         # enable interrupts
6599         local $SIG{INT} =
6600             local $SIG{TERM} =
6601             local $SIG{QUIT} =
6602             local $SIG{HUP} =
6603             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6604         local $SIG{ALRM} = sub { die "got timeout\n"; };
6605
6606         $oldtimeout = alarm($timeout);
6607
6608         my $parser = sub {
6609             my $line = shift;
6610
6611             print "$line\n";
6612
6613             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6614                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6615                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6616             } elsif ($line =~ m/^CTIME: /) {
6617                 # we correctly received the vma config, so we can disable
6618                 # the timeout now for disk allocation (set to 10 minutes, so
6619                 # that we always timeout if something goes wrong)
6620                 alarm(600);
6621                 &$print_devmap();
6622                 print $fifofh "done\n";
6623                 my $tmp = $oldtimeout || 0;
6624                 $oldtimeout = undef;
6625                 alarm($tmp);
6626                 close($fifofh);
6627             }
6628         };
6629
6630         print "restore vma archive: $dbg_cmdstring\n";
6631         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6632     };
6633     my $err = $@;
6634
6635     alarm($oldtimeout) if $oldtimeout;
6636
6637     my $vollist = [];
6638     foreach my $devname (keys %$devinfo) {
6639         my $volid = $devinfo->{$devname}->{volid};
6640         push @$vollist, $volid if $volid;
6641     }
6642
6643     PVE::Storage::deactivate_volumes($cfg, $vollist);
6644
6645     unlink $mapfifo;
6646
6647     if ($err) {
6648         rmtree $tmpdir;
6649         unlink $tmpfn;
6650
6651         foreach my $devname (keys %$devinfo) {
6652             my $volid = $devinfo->{$devname}->{volid};
6653             next if !$volid;
6654             eval {
6655                 if ($volid =~ m|^/|) {
6656                     unlink $volid || die 'unlink failed\n';
6657                 } else {
6658                     PVE::Storage::vdisk_free($cfg, $volid);
6659                 }
6660                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6661             };
6662             print STDERR "unable to cleanup '$volid' - $@" if $@;
6663         }
6664         die $err;
6665     }
6666
6667     rmtree $tmpdir;
6668
6669     rename($tmpfn, $conffile) ||
6670         die "unable to commit configuration file '$conffile'\n";
6671
6672     PVE::Cluster::cfs_update(); # make sure we read new file
6673
6674     eval { rescan($vmid, 1); };
6675     warn $@ if $@;
6676 }
6677
6678 sub restore_tar_archive {
6679     my ($archive, $vmid, $user, $opts) = @_;
6680
6681     if ($archive ne '-') {
6682         my $firstfile = tar_archive_read_firstfile($archive);
6683         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6684             if $firstfile ne 'qemu-server.conf';
6685     }
6686
6687     my $storecfg = PVE::Storage::config();
6688
6689     # destroy existing data - keep empty config
6690     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6691     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6692
6693     my $tocmd = "/usr/lib/qemu-server/qmextract";
6694
6695     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6696     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6697     $tocmd .= ' --prealloc' if $opts->{prealloc};
6698     $tocmd .= ' --info' if $opts->{info};
6699
6700     # tar option "xf" does not autodetect compression when read from STDIN,
6701     # so we pipe to zcat
6702     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6703         PVE::Tools::shellquote("--to-command=$tocmd");
6704
6705     my $tmpdir = "/var/tmp/vzdumptmp$$";
6706     mkpath $tmpdir;
6707
6708     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6709     local $ENV{VZDUMP_VMID} = $vmid;
6710     local $ENV{VZDUMP_USER} = $user;
6711
6712     my $conffile = PVE::QemuConfig->config_file($vmid);
6713     my $tmpfn = "$conffile.$$.tmp";
6714
6715     # disable interrupts (always do cleanups)
6716     local $SIG{INT} =
6717         local $SIG{TERM} =
6718         local $SIG{QUIT} =
6719         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6720
6721     eval {
6722         # enable interrupts
6723         local $SIG{INT} =
6724             local $SIG{TERM} =
6725             local $SIG{QUIT} =
6726             local $SIG{HUP} =
6727             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6728
6729         if ($archive eq '-') {
6730             print "extracting archive from STDIN\n";
6731             run_command($cmd, input => "<&STDIN");
6732         } else {
6733             print "extracting archive '$archive'\n";
6734             run_command($cmd);
6735         }
6736
6737         return if $opts->{info};
6738
6739         # read new mapping
6740         my $map = {};
6741         my $statfile = "$tmpdir/qmrestore.stat";
6742         if (my $fd = IO::File->new($statfile, "r")) {
6743             while (defined (my $line = <$fd>)) {
6744                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6745                     $map->{$1} = $2 if $1;
6746                 } else {
6747                     print STDERR "unable to parse line in statfile - $line\n";
6748                 }
6749             }
6750             $fd->close();
6751         }
6752
6753         my $confsrc = "$tmpdir/qemu-server.conf";
6754
6755         my $srcfd = new IO::File($confsrc, "r") ||
6756             die "unable to open file '$confsrc'\n";
6757
6758         my $outfd = new IO::File ($tmpfn, "w") ||
6759             die "unable to write config for VM $vmid\n";
6760
6761         my $cookie = { netcount => 0 };
6762         while (defined (my $line = <$srcfd>)) {
6763             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6764         }
6765
6766         $srcfd->close();
6767         $outfd->close();
6768     };
6769     my $err = $@;
6770
6771     if ($err) {
6772
6773         unlink $tmpfn;
6774
6775         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6776
6777         die $err;
6778     }
6779
6780     rmtree $tmpdir;
6781
6782     rename $tmpfn, $conffile ||
6783         die "unable to commit configuration file '$conffile'\n";
6784
6785     PVE::Cluster::cfs_update(); # make sure we read new file
6786
6787     eval { rescan($vmid, 1); };
6788     warn $@ if $@;
6789 };
6790
6791 sub foreach_storage_used_by_vm {
6792     my ($conf, $func) = @_;
6793
6794     my $sidhash = {};
6795
6796     foreach_drive($conf, sub {
6797         my ($ds, $drive) = @_;
6798         return if drive_is_cdrom($drive);
6799
6800         my $volid = $drive->{file};
6801
6802         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6803         $sidhash->{$sid} = $sid if $sid;
6804     });
6805
6806     foreach my $sid (sort keys %$sidhash) {
6807         &$func($sid);
6808     }
6809 }
6810
6811 sub do_snapshots_with_qemu {
6812     my ($storecfg, $volid) = @_;
6813
6814     my $storage_name = PVE::Storage::parse_volume_id($volid);
6815     my $scfg = $storecfg->{ids}->{$storage_name};
6816
6817     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6818         return 1;
6819     }
6820
6821     if ($volid =~ m/\.(qcow2|qed)$/){
6822         return 1;
6823     }
6824
6825     return undef;
6826 }
6827
6828 sub qga_check_running {
6829     my ($vmid, $nowarn) = @_;
6830
6831     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6832     if ($@) {
6833         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6834         return 0;
6835     }
6836     return 1;
6837 }
6838
6839 sub template_create {
6840     my ($vmid, $conf, $disk) = @_;
6841
6842     my $storecfg = PVE::Storage::config();
6843
6844     foreach_drive($conf, sub {
6845         my ($ds, $drive) = @_;
6846
6847         return if drive_is_cdrom($drive);
6848         return if $disk && $ds ne $disk;
6849
6850         my $volid = $drive->{file};
6851         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6852
6853         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6854         $drive->{file} = $voliddst;
6855         $conf->{$ds} = print_drive($vmid, $drive);
6856         PVE::QemuConfig->write_config($vmid, $conf);
6857     });
6858 }
6859
6860 sub convert_iscsi_path {
6861     my ($path) = @_;
6862
6863     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6864         my $portal = $1;
6865         my $target = $2;
6866         my $lun = $3;
6867
6868         my $initiator_name = get_initiator_name();
6869
6870         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6871                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6872     }
6873
6874     die "cannot convert iscsi path '$path', unkown format\n";
6875 }
6876
6877 sub qemu_img_convert {
6878     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6879
6880     my $storecfg = PVE::Storage::config();
6881     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6882     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6883
6884     if ($src_storeid && $dst_storeid) {
6885
6886         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6887
6888         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6889         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6890
6891         my $src_format = qemu_img_format($src_scfg, $src_volname);
6892         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6893
6894         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6895         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6896
6897         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6898         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6899
6900         my $cmd = [];
6901         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6902         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6903         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6904         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6905
6906         if ($src_is_iscsi) {
6907             push @$cmd, '--image-opts';
6908             $src_path = convert_iscsi_path($src_path);
6909         } else {
6910             push @$cmd, '-f', $src_format;
6911         }
6912
6913         if ($dst_is_iscsi) {
6914             push @$cmd, '--target-image-opts';
6915             $dst_path = convert_iscsi_path($dst_path);
6916         } else {
6917             push @$cmd, '-O', $dst_format;
6918         }
6919
6920         push @$cmd, $src_path;
6921
6922         if (!$dst_is_iscsi && $is_zero_initialized) {
6923             push @$cmd, "zeroinit:$dst_path";
6924         } else {
6925             push @$cmd, $dst_path;
6926         }
6927
6928         my $parser = sub {
6929             my $line = shift;
6930             if($line =~ m/\((\S+)\/100\%\)/){
6931                 my $percent = $1;
6932                 my $transferred = int($size * $percent / 100);
6933                 my $remaining = $size - $transferred;
6934
6935                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6936             }
6937
6938         };
6939
6940         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6941         my $err = $@;
6942         die "copy failed: $err" if $err;
6943     }
6944 }
6945
6946 sub qemu_img_format {
6947     my ($scfg, $volname) = @_;
6948
6949     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6950         return $1;
6951     } else {
6952         return "raw";
6953     }
6954 }
6955
6956 sub qemu_drive_mirror {
6957     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6958
6959     $jobs = {} if !$jobs;
6960
6961     my $qemu_target;
6962     my $format;
6963     $jobs->{"drive-$drive"} = {};
6964
6965     if ($dst_volid =~ /^nbd:/) {
6966         $qemu_target = $dst_volid;
6967         $format = "nbd";
6968     } else {
6969         my $storecfg = PVE::Storage::config();
6970         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6971
6972         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6973
6974         $format = qemu_img_format($dst_scfg, $dst_volname);
6975
6976         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6977
6978         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6979     }
6980
6981     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6982     $opts->{format} = $format if $format;
6983
6984     if (defined($bwlimit)) {
6985         $opts->{speed} = $bwlimit * 1024;
6986         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6987     } else {
6988         print "drive mirror is starting for drive-$drive\n";
6989     }
6990
6991     # if a job already runs for this device we get an error, catch it for cleanup
6992     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6993     if (my $err = $@) {
6994         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6995         warn "$@\n" if $@;
6996         die "mirroring error: $err\n";
6997     }
6998
6999     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
7000 }
7001
7002 sub qemu_drive_mirror_monitor {
7003     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
7004
7005     eval {
7006         my $err_complete = 0;
7007
7008         while (1) {
7009             die "storage migration timed out\n" if $err_complete > 300;
7010
7011             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7012
7013             my $running_mirror_jobs = {};
7014             foreach my $stat (@$stats) {
7015                 next if $stat->{type} ne 'mirror';
7016                 $running_mirror_jobs->{$stat->{device}} = $stat;
7017             }
7018
7019             my $readycounter = 0;
7020
7021             foreach my $job (keys %$jobs) {
7022
7023                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
7024                     print "$job : finished\n";
7025                     delete $jobs->{$job};
7026                     next;
7027                 }
7028
7029                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7030
7031                 my $busy = $running_mirror_jobs->{$job}->{busy};
7032                 my $ready = $running_mirror_jobs->{$job}->{ready};
7033                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7034                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7035                     my $remaining = $total - $transferred;
7036                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7037
7038                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7039                 }
7040
7041                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7042             }
7043
7044             last if scalar(keys %$jobs) == 0;
7045
7046             if ($readycounter == scalar(keys %$jobs)) {
7047                 print "all mirroring jobs are ready \n";
7048                 last if $skipcomplete; #do the complete later
7049
7050                 if ($vmiddst && $vmiddst != $vmid) {
7051                     my $agent_running = $qga && qga_check_running($vmid);
7052                     if ($agent_running) {
7053                         print "freeze filesystem\n";
7054                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7055                     } else {
7056                         print "suspend vm\n";
7057                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7058                     }
7059
7060                     # if we clone a disk for a new target vm, we don't switch the disk
7061                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7062
7063                     if ($agent_running) {
7064                         print "unfreeze filesystem\n";
7065                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7066                     } else {
7067                         print "resume vm\n";
7068                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7069                     }
7070
7071                     last;
7072                 } else {
7073
7074                     foreach my $job (keys %$jobs) {
7075                         # try to switch the disk if source and destination are on the same guest
7076                         print "$job: Completing block job...\n";
7077
7078                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7079                         if ($@ =~ m/cannot be completed/) {
7080                             print "$job: Block job cannot be completed, try again.\n";
7081                             $err_complete++;
7082                         }else {
7083                             print "$job: Completed successfully.\n";
7084                             $jobs->{$job}->{complete} = 1;
7085                         }
7086                     }
7087                 }
7088             }
7089             sleep 1;
7090         }
7091     };
7092     my $err = $@;
7093
7094     if ($err) {
7095         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7096         die "mirroring error: $err";
7097     }
7098
7099 }
7100
7101 sub qemu_blockjobs_cancel {
7102     my ($vmid, $jobs) = @_;
7103
7104     foreach my $job (keys %$jobs) {
7105         print "$job: Cancelling block job\n";
7106         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7107         $jobs->{$job}->{cancel} = 1;
7108     }
7109
7110     while (1) {
7111         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7112
7113         my $running_jobs = {};
7114         foreach my $stat (@$stats) {
7115             $running_jobs->{$stat->{device}} = $stat;
7116         }
7117
7118         foreach my $job (keys %$jobs) {
7119
7120             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7121                 print "$job: Done.\n";
7122                 delete $jobs->{$job};
7123             }
7124         }
7125
7126         last if scalar(keys %$jobs) == 0;
7127
7128         sleep 1;
7129     }
7130 }
7131
7132 sub clone_disk {
7133     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7134         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7135
7136     my $newvolid;
7137
7138     if (!$full) {
7139         print "create linked clone of drive $drivename ($drive->{file})\n";
7140         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7141         push @$newvollist, $newvolid;
7142     } else {
7143
7144         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7145         $storeid = $storage if $storage;
7146
7147         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7148         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7149
7150         print "create full clone of drive $drivename ($drive->{file})\n";
7151         my $name = undef;
7152         if (drive_is_cloudinit($drive)) {
7153             $name = "vm-$newvmid-cloudinit";
7154             $snapname = undef;
7155             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7156             if ($dst_format ne 'raw') {
7157                 $name .= ".$dst_format";
7158             }
7159         }
7160         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7161         push @$newvollist, $newvolid;
7162
7163         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7164
7165         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7166         if (!$running || $snapname) {
7167             # TODO: handle bwlimits
7168             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7169         } else {
7170
7171             my $kvmver = get_running_qemu_version ($vmid);
7172             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7173                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7174                     if $drive->{iothread};
7175             }
7176
7177             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7178         }
7179     }
7180
7181     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7182
7183     my $disk = $drive;
7184     $disk->{format} = undef;
7185     $disk->{file} = $newvolid;
7186     $disk->{size} = $size;
7187
7188     return $disk;
7189 }
7190
7191 # this only works if VM is running
7192 sub get_current_qemu_machine {
7193     my ($vmid) = @_;
7194
7195     my $cmd = { execute => 'query-machines', arguments => {} };
7196     my $res = vm_qmp_command($vmid, $cmd);
7197
7198     my ($current, $default);
7199     foreach my $e (@$res) {
7200         $default = $e->{name} if $e->{'is-default'};
7201         $current = $e->{name} if $e->{'is-current'};
7202     }
7203
7204     # fallback to the default machine if current is not supported by qemu
7205     return $current || $default || 'pc';
7206 }
7207
7208 sub get_running_qemu_version {
7209     my ($vmid) = @_;
7210     my $cmd = { execute => 'query-version', arguments => {} };
7211     my $res = vm_qmp_command($vmid, $cmd);
7212     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7213 }
7214
7215 sub qemu_machine_feature_enabled {
7216     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7217
7218     my $current_major;
7219     my $current_minor;
7220
7221     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7222
7223         $current_major = $3;
7224         $current_minor = $4;
7225
7226     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7227
7228         $current_major = $1;
7229         $current_minor = $2;
7230     }
7231
7232     return 1 if $current_major > $version_major ||
7233                 ($current_major == $version_major &&
7234                  $current_minor >= $version_minor);
7235 }
7236
7237 sub qemu_machine_pxe {
7238     my ($vmid, $conf) = @_;
7239
7240     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7241
7242     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7243         $machine .= '.pxe';
7244     }
7245
7246     return $machine;
7247 }
7248
7249 sub qemu_use_old_bios_files {
7250     my ($machine_type) = @_;
7251
7252     return if !$machine_type;
7253
7254     my $use_old_bios_files = undef;
7255
7256     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7257         $machine_type = $1;
7258         $use_old_bios_files = 1;
7259     } else {
7260         my $kvmver = kvm_user_version();
7261         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7262         # load new efi bios files on migration. So this hack is required to allow
7263         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7264         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7265         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7266     }
7267
7268     return ($use_old_bios_files, $machine_type);
7269 }
7270
7271 sub create_efidisk($$$$$) {
7272     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7273
7274     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7275     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7276
7277     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7278     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7279     PVE::Storage::activate_volumes($storecfg, [$volid]);
7280
7281     my $path = PVE::Storage::path($storecfg, $volid);
7282     eval {
7283         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7284     };
7285     die "Copying EFI vars image failed: $@" if $@;
7286
7287     return ($volid, $vars_size);
7288 }
7289
7290 sub vm_iothreads_list {
7291     my ($vmid) = @_;
7292
7293     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7294
7295     my $iothreads = {};
7296     foreach my $iothread (@$res) {
7297         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7298     }
7299
7300     return $iothreads;
7301 }
7302
7303 sub scsihw_infos {
7304     my ($conf, $drive) = @_;
7305
7306     my $maxdev = 0;
7307
7308     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7309         $maxdev = 7;
7310     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7311         $maxdev = 1;
7312     } else {
7313         $maxdev = 256;
7314     }
7315
7316     my $controller = int($drive->{index} / $maxdev);
7317     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7318
7319     return ($maxdev, $controller, $controller_prefix);
7320 }
7321
7322 sub add_hyperv_enlightenments {
7323     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7324
7325     return if $winversion < 6;
7326     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7327
7328     if ($gpu_passthrough || defined($hv_vendor_id)) {
7329         $hv_vendor_id //= 'proxmox';
7330         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7331     }
7332
7333     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7334         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7335         push @$cpuFlags , 'hv_vapic';
7336         push @$cpuFlags , 'hv_time';
7337     } else {
7338         push @$cpuFlags , 'hv_spinlocks=0xffff';
7339     }
7340
7341     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7342         push @$cpuFlags , 'hv_reset';
7343         push @$cpuFlags , 'hv_vpindex';
7344         push @$cpuFlags , 'hv_runtime';
7345     }
7346
7347     if ($winversion >= 7) {
7348         push @$cpuFlags , 'hv_relaxed';
7349
7350         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7351             push @$cpuFlags , 'hv_synic';
7352             push @$cpuFlags , 'hv_stimer';
7353         }
7354
7355         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7356             push @$cpuFlags , 'hv_ipi';
7357         }
7358     }
7359 }
7360
7361 sub windows_version {
7362     my ($ostype) = @_;
7363
7364     return 0 if !$ostype;
7365
7366     my $winversion = 0;
7367
7368     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7369         $winversion = 5;
7370     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7371         $winversion = 6;
7372     } elsif ($ostype =~ m/^win(\d+)$/) {
7373         $winversion = $1;
7374     }
7375
7376     return $winversion;
7377 }
7378
7379 sub resolve_dst_disk_format {
7380         my ($storecfg, $storeid, $src_volname, $format) = @_;
7381         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7382
7383         if (!$format) {
7384             # if no target format is specified, use the source disk format as hint
7385             if ($src_volname) {
7386                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7387                 $format = qemu_img_format($scfg, $src_volname);
7388             } else {
7389                 return $defFormat;
7390             }
7391         }
7392
7393         # test if requested format is supported - else use default
7394         my $supported = grep { $_ eq $format } @$validFormats;
7395         $format = $defFormat if !$supported;
7396         return $format;
7397 }
7398
7399 sub resolve_first_disk {
7400     my $conf = shift;
7401     my @disks = PVE::QemuServer::valid_drive_names();
7402     my $firstdisk;
7403     foreach my $ds (reverse @disks) {
7404         next if !$conf->{$ds};
7405         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7406         next if PVE::QemuServer::drive_is_cdrom($disk);
7407         $firstdisk = $ds;
7408     }
7409     return $firstdisk;
7410 }
7411
7412 sub generate_uuid {
7413     my ($uuid, $uuid_str);
7414     UUID::generate($uuid);
7415     UUID::unparse($uuid, $uuid_str);
7416     return $uuid_str;
7417 }
7418
7419 sub generate_smbios1_uuid {
7420     return "uuid=".generate_uuid();
7421 }
7422
7423 sub nbd_stop {
7424     my ($vmid) = @_;
7425
7426     vm_mon_cmd($vmid, 'nbd-server-stop');
7427 }
7428
7429 sub create_reboot_request {
7430     my ($vmid) = @_;
7431     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7432         or die "failed to create reboot trigger file: $!\n";
7433     close($fh);
7434 }
7435
7436 sub clear_reboot_request {
7437     my ($vmid) = @_;
7438     my $path = "/run/qemu-server/$vmid.reboot";
7439     my $res = 0;
7440
7441     $res = unlink($path);
7442     die "could not remove reboot request for $vmid: $!"
7443         if !$res && $! != POSIX::ENOENT;
7444
7445     return $res;
7446 }
7447
7448 # bash completion helper
7449
7450 sub complete_backup_archives {
7451     my ($cmdname, $pname, $cvalue) = @_;
7452
7453     my $cfg = PVE::Storage::config();
7454
7455     my $storeid;
7456
7457     if ($cvalue =~ m/^([^:]+):/) {
7458         $storeid = $1;
7459     }
7460
7461     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7462
7463     my $res = [];
7464     foreach my $id (keys %$data) {
7465         foreach my $item (@{$data->{$id}}) {
7466             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7467             push @$res, $item->{volid} if defined($item->{volid});
7468         }
7469     }
7470
7471     return $res;
7472 }
7473
7474 my $complete_vmid_full = sub {
7475     my ($running) = @_;
7476
7477     my $idlist = vmstatus();
7478
7479     my $res = [];
7480
7481     foreach my $id (keys %$idlist) {
7482         my $d = $idlist->{$id};
7483         if (defined($running)) {
7484             next if $d->{template};
7485             next if $running && $d->{status} ne 'running';
7486             next if !$running && $d->{status} eq 'running';
7487         }
7488         push @$res, $id;
7489
7490     }
7491     return $res;
7492 };
7493
7494 sub complete_vmid {
7495     return &$complete_vmid_full();
7496 }
7497
7498 sub complete_vmid_stopped {
7499     return &$complete_vmid_full(0);
7500 }
7501
7502 sub complete_vmid_running {
7503     return &$complete_vmid_full(1);
7504 }
7505
7506 sub complete_storage {
7507
7508     my $cfg = PVE::Storage::config();
7509     my $ids = $cfg->{ids};
7510
7511     my $res = [];
7512     foreach my $sid (keys %$ids) {
7513         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7514         next if !$ids->{$sid}->{content}->{images};
7515         push @$res, $sid;
7516     }
7517
7518     return $res;
7519 }
7520
7521 1;