PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $pcisysfs = "/sys/bus/pci";
 123
 124 my $cpu_vendor_list = {
 125     # Intel CPUs
 126     486 => 'GenuineIntel',
 127     pentium => 'GenuineIntel',
 128     pentium2  => 'GenuineIntel',
 129     pentium3  => 'GenuineIntel',
 130     coreduo => 'GenuineIntel',
 131     core2duo => 'GenuineIntel',
 132     Conroe  => 'GenuineIntel',
 133     Penryn  => 'GenuineIntel',
 134     Nehalem  => 'GenuineIntel',
 135     'Nehalem-IBRS'  => 'GenuineIntel',
 136     Westmere => 'GenuineIntel',
 137     'Westmere-IBRS' => 'GenuineIntel',
 138     SandyBridge => 'GenuineIntel',
 139     'SandyBridge-IBRS' => 'GenuineIntel',
 140     IvyBridge => 'GenuineIntel',
 141     'IvyBridge-IBRS' => 'GenuineIntel',
 142     Haswell => 'GenuineIntel',
 143     'Haswell-IBRS' => 'GenuineIntel',
 144     'Haswell-noTSX' => 'GenuineIntel',
 145     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 146     Broadwell => 'GenuineIntel',
 147     'Broadwell-IBRS' => 'GenuineIntel',
 148     'Broadwell-noTSX' => 'GenuineIntel',
 149     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 150     'Skylake-Client' => 'GenuineIntel',
 151     'Skylake-Client-IBRS' => 'GenuineIntel',
 152     'Skylake-Server' => 'GenuineIntel',
 153     'Skylake-Server-IBRS' => 'GenuineIntel',
 154
 155     # AMD CPUs
 156     athlon => 'AuthenticAMD',
 157     phenom  => 'AuthenticAMD',
 158     Opteron_G1  => 'AuthenticAMD',
 159     Opteron_G2  => 'AuthenticAMD',
 160     Opteron_G3  => 'AuthenticAMD',
 161     Opteron_G4  => 'AuthenticAMD',
 162     Opteron_G5  => 'AuthenticAMD',
 163     EPYC => 'AuthenticAMD',
 164     'EPYC-IBPB' => 'AuthenticAMD',
 165
 166     # generic types, use vendor from host node
 167     host => 'default',
 168     kvm32 => 'default',
 169     kvm64 => 'default',
 170     qemu32 => 'default',
 171     qemu64 => 'default',
 172     max => 'default',
 173 };
 174
 175 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 176
 177 my $cpu_fmt = {
 178     cputype => {
 179         description => "Emulated CPU type.",
 180         type => 'string',
 181         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 182         default => 'kvm64',
 183         default_key => 1,
 184     },
 185     hidden => {
 186         description => "Do not identify as a KVM virtual machine.",
 187         type => 'boolean',
 188         optional => 1,
 189         default => 0
 190     },
 191     flags => {
 192         description => "List of additional CPU flags separated by ';'."
 193                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 194                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 195         format_description => '+FLAG[;-FLAG...]',
 196         type => 'string',
 197         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 198         optional => 1,
 199     },
 200 };
 201
 202 my $watchdog_fmt = {
 203     model => {
 204         default_key => 1,
 205         type => 'string',
 206         enum => [qw(i6300esb ib700)],
 207         description => "Watchdog type to emulate.",
 208         default => 'i6300esb',
 209         optional => 1,
 210     },
 211     action => {
 212         type => 'string',
 213         enum => [qw(reset shutdown poweroff pause debug none)],
 214         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 215         optional => 1,
 216     },
 217 };
 218 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 219
 220 my $agent_fmt = {
 221     enabled => {
 222         description => "Enable/disable Qemu GuestAgent.",
 223         type => 'boolean',
 224         default => 0,
 225         default_key => 1,
 226     },
 227     fstrim_cloned_disks => {
 228         description => "Run fstrim after cloning/moving a disk.",
 229         type => 'boolean',
 230         optional => 1,
 231         default => 0
 232     },
 233 };
 234
 235 my $vga_fmt = {
 236     type => {
 237         description => "Select the VGA type.",
 238         type => 'string',
 239         default => 'std',
 240         optional => 1,
 241         default_key => 1,
 242         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 243     },
 244     memory => {
 245         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 246         type => 'integer',
 247         optional => 1,
 248         minimum => 4,
 249         maximum => 512,
 250     },
 251 };
 252
 253 my $confdesc = {
 254     onboot => {
 255         optional => 1,
 256         type => 'boolean',
 257         description => "Specifies whether a VM will be started during system bootup.",
 258         default => 0,
 259     },
 260     autostart => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Automatic restart after crash (currently ignored).",
 264         default => 0,
 265     },
 266     hotplug => {
 267         optional => 1,
 268         type => 'string', format => 'pve-hotplug-features',
 269         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 270         default => 'network,disk,usb',
 271     },
 272     reboot => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 276         default => 1,
 277     },
 278     lock => {
 279         optional => 1,
 280         type => 'string',
 281         description => "Lock/unlock the VM.",
 282         enum => [qw(migrate backup snapshot rollback)],
 283     },
 284     cpulimit => {
 285         optional => 1,
 286         type => 'number',
 287         description => "Limit of CPU usage.",
 288         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 289         minimum => 0,
 290         maximum => 128,
 291         default => 0,
 292     },
 293     cpuunits => {
 294         optional => 1,
 295         type => 'integer',
 296         description => "CPU weight for a VM.",
 297         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 298         minimum => 2,
 299         maximum => 262144,
 300         default => 1024,
 301     },
 302     memory => {
 303         optional => 1,
 304         type => 'integer',
 305         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 306         minimum => 16,
 307         default => 512,
 308     },
 309     balloon => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 313         minimum => 0,
 314     },
 315     shares => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 319         minimum => 0,
 320         maximum => 50000,
 321         default => 1000,
 322     },
 323     keyboard => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 327                        "It should not be necessary to set it.",
 328         enum => PVE::Tools::kvmkeymaplist(),
 329         default => undef,
 330     },
 331     name => {
 332         optional => 1,
 333         type => 'string', format => 'dns-name',
 334         description => "Set a name for the VM. Only used on the configuration web interface.",
 335     },
 336     scsihw => {
 337         optional => 1,
 338         type => 'string',
 339         description => "SCSI controller model",
 340         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 341         default => 'lsi',
 342     },
 343     description => {
 344         optional => 1,
 345         type => 'string',
 346         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 347     },
 348     ostype => {
 349         optional => 1,
 350         type => 'string',
 351         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 352         description => "Specify guest operating system.",
 353         verbose_description => <<EODESC,
 354 Specify guest operating system. This is used to enable special
 355 optimization/features for specific operating systems:
 356
 357 [horizontal]
 358 other;; unspecified OS
 359 wxp;; Microsoft Windows XP
 360 w2k;; Microsoft Windows 2000
 361 w2k3;; Microsoft Windows 2003
 362 w2k8;; Microsoft Windows 2008
 363 wvista;; Microsoft Windows Vista
 364 win7;; Microsoft Windows 7
 365 win8;; Microsoft Windows 8/2012/2012r2
 366 win10;; Microsoft Windows 10/2016
 367 l24;; Linux 2.4 Kernel
 368 l26;; Linux 2.6/3.X Kernel
 369 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 370 EODESC
 371     },
 372     boot => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 376         pattern => '[acdn]{1,4}',
 377         default => 'cdn',
 378     },
 379     bootdisk => {
 380         optional => 1,
 381         type => 'string', format => 'pve-qm-bootdisk',
 382         description => "Enable booting from specified disk.",
 383         pattern => '(ide|sata|scsi|virtio)\d+',
 384     },
 385     smp => {
 386         optional => 1,
 387         type => 'integer',
 388         description => "The number of CPUs. Please use option -sockets instead.",
 389         minimum => 1,
 390         default => 1,
 391     },
 392     sockets => {
 393         optional => 1,
 394         type => 'integer',
 395         description => "The number of CPU sockets.",
 396         minimum => 1,
 397         default => 1,
 398     },
 399     cores => {
 400         optional => 1,
 401         type => 'integer',
 402         description => "The number of cores per socket.",
 403         minimum => 1,
 404         default => 1,
 405     },
 406     numa => {
 407         optional => 1,
 408         type => 'boolean',
 409         description => "Enable/disable NUMA.",
 410         default => 0,
 411     },
 412     hugepages => {
 413         optional => 1,
 414         type => 'string',
 415         description => "Enable/disable hugepages memory.",
 416         enum => [qw(any 2 1024)],
 417     },
 418     vcpus => {
 419         optional => 1,
 420         type => 'integer',
 421         description => "Number of hotplugged vcpus.",
 422         minimum => 1,
 423         default => 0,
 424     },
 425     acpi => {
 426         optional => 1,
 427         type => 'boolean',
 428         description => "Enable/disable ACPI.",
 429         default => 1,
 430     },
 431     agent => {
 432         optional => 1,
 433         description => "Enable/disable Qemu GuestAgent and its properties.",
 434         type => 'string',
 435         format => $agent_fmt,
 436     },
 437     kvm => {
 438         optional => 1,
 439         type => 'boolean',
 440         description => "Enable/disable KVM hardware virtualization.",
 441         default => 1,
 442     },
 443     tdf => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable time drift fix.",
 447         default => 0,
 448     },
 449     localtime => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 453     },
 454     freeze => {
 455         optional => 1,
 456         type => 'boolean',
 457         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 458     },
 459     vga => {
 460         optional => 1,
 461         type => 'string', format => $vga_fmt,
 462         description => "Configure the VGA hardware.",
 463         verbose_description => "Configure the VGA Hardware. If you want to use ".
 464             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 465             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 466             "is 'std' for all OS types besides some Windows versions (XP and " .
 467             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 468             "display server. For win* OS you can select how many independent " .
 469             "displays you want, Linux guests can add displays them self.\n".
 470             "You can also run without any graphic card, using a serial device as terminal.",
 471     },
 472     watchdog => {
 473         optional => 1,
 474         type => 'string', format => 'pve-qm-watchdog',
 475         description => "Create a virtual hardware watchdog device.",
 476         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 477             " (by a guest action), the watchdog must be periodically polled " .
 478             "by an agent inside the guest or else the watchdog will reset " .
 479             "the guest (or execute the respective action specified)",
 480     },
 481     startdate => {
 482         optional => 1,
 483         type => 'string',
 484         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 485         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 486         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 487         default => 'now',
 488     },
 489     startup =>  get_standard_option('pve-startup-order'),
 490     template => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable Template.",
 494         default => 0,
 495     },
 496     args => {
 497         optional => 1,
 498         type => 'string',
 499         description => "Arbitrary arguments passed to kvm.",
 500         verbose_description => <<EODESCR,
 501 Arbitrary arguments passed to kvm, for example:
 502
 503 args: -no-reboot -no-hpet
 504
 505 NOTE: this option is for experts only.
 506 EODESCR
 507     },
 508     tablet => {
 509         optional => 1,
 510         type => 'boolean',
 511         default => 1,
 512         description => "Enable/disable the USB tablet device.",
 513         verbose_description => "Enable/disable the USB tablet device. This device is " .
 514             "usually needed to allow absolute mouse positioning with VNC. " .
 515             "Else the mouse runs out of sync with normal VNC clients. " .
 516             "If you're running lots of console-only guests on one host, " .
 517             "you may consider disabling this to save some context switches. " .
 518             "This is turned off by default if you use spice (-vga=qxl).",
 519     },
 520     migrate_speed => {
 521         optional => 1,
 522         type => 'integer',
 523         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 524         minimum => 0,
 525         default => 0,
 526     },
 527     migrate_downtime => {
 528         optional => 1,
 529         type => 'number',
 530         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 531         minimum => 0,
 532         default => 0.1,
 533     },
 534     cdrom => {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-ide',
 537         typetext => '<volume>',
 538         description => "This is an alias for option -ide2",
 539     },
 540     cpu => {
 541         optional => 1,
 542         description => "Emulated CPU type.",
 543         type => 'string',
 544         format => $cpu_fmt,
 545     },
 546     parent => get_standard_option('pve-snapshot-name', {
 547         optional => 1,
 548         description => "Parent snapshot name. This is used internally, and should not be modified.",
 549     }),
 550     snaptime => {
 551         optional => 1,
 552         description => "Timestamp for snapshots.",
 553         type => 'integer',
 554         minimum => 0,
 555     },
 556     vmstate => {
 557         optional => 1,
 558         type => 'string', format => 'pve-volume-id',
 559         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 560     },
 561     vmstatestorage => get_standard_option('pve-storage-id', {
 562         description => "Default storage for VM state volumes/files.",
 563         optional => 1,
 564     }),
 565     runningmachine => get_standard_option('pve-qemu-machine', {
 566         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 567     }),
 568     machine => get_standard_option('pve-qemu-machine'),
 569     arch => {
 570         description => "Virtual processor architecture. Defaults to the host.",
 571         optional => 1,
 572         type => 'string',
 573         enum => [qw(x86_64 aarch64)],
 574     },
 575     smbios1 => {
 576         description => "Specify SMBIOS type 1 fields.",
 577         type => 'string', format => 'pve-qm-smbios1',
 578         maxLength => 256,
 579         optional => 1,
 580     },
 581     protection => {
 582         optional => 1,
 583         type => 'boolean',
 584         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 585         default => 0,
 586     },
 587     bios => {
 588         optional => 1,
 589         type => 'string',
 590         enum => [ qw(seabios ovmf) ],
 591         description => "Select BIOS implementation.",
 592         default => 'seabios',
 593     },
 594     vmgenid => {
 595         type => 'string',
 596         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 597         format_description => 'UUID',
 598         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 599         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 600             " 128-bit integer value identifier to the guest OS. This allows to".
 601             " notify the guest operating system when the virtual machine is".
 602             " executed with a different configuration (e.g. snapshot execution".
 603             " or creation from a template). The guest operating system notices".
 604             " the change, and is then able to react as appropriate by marking".
 605             " its copies of distributed databases as dirty, re-initializing its".
 606             " random number generator, etc.\n".
 607             "Note that auto-creation only works when done throug API/CLI create".
 608             " or update methods, but not when manually editing the config file.",
 609         default => "1 (autogenerated)",
 610         optional => 1,
 611     },
 612 };
 613
 614 my $confdesc_cloudinit = {
 615     citype => {
 616         optional => 1,
 617         type => 'string',
 618         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 619         enum => ['configdrive2', 'nocloud'],
 620     },
 621     ciuser => {
 622         optional => 1,
 623         type => 'string',
 624         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 625     },
 626     cipassword => {
 627         optional => 1,
 628         type => 'string',
 629         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 630     },
 631     searchdomain => {
 632         optional => 1,
 633         type => 'string',
 634         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 635     },
 636     nameserver => {
 637         optional => 1,
 638         type => 'string', format => 'address-list',
 639         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 640     },
 641     sshkeys => {
 642         optional => 1,
 643         type => 'string',
 644         format => 'urlencoded',
 645         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 646     },
 647 };
 648
 649 # what about other qemu settings ?
 650 #cpu => 'string',
 651 #machine => 'string',
 652 #fda => 'file',
 653 #fdb => 'file',
 654 #mtdblock => 'file',
 655 #sd => 'file',
 656 #pflash => 'file',
 657 #snapshot => 'bool',
 658 #bootp => 'file',
 659 ##tftp => 'dir',
 660 ##smb => 'dir',
 661 #kernel => 'file',
 662 #append => 'string',
 663 #initrd => 'file',
 664 ##soundhw => 'string',
 665
 666 while (my ($k, $v) = each %$confdesc) {
 667     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 668 }
 669
 670 my $MAX_IDE_DISKS = 4;
 671 my $MAX_SCSI_DISKS = 14;
 672 my $MAX_VIRTIO_DISKS = 16;
 673 my $MAX_SATA_DISKS = 6;
 674 my $MAX_USB_DEVICES = 5;
 675 my $MAX_NETS = 32;
 676 my $MAX_UNUSED_DISKS = 256;
 677 my $MAX_HOSTPCI_DEVICES = 4;
 678 my $MAX_SERIAL_PORTS = 4;
 679 my $MAX_PARALLEL_PORTS = 3;
 680 my $MAX_NUMA = 8;
 681
 682 my $numa_fmt = {
 683     cpus => {
 684         type => "string",
 685         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 686         description => "CPUs accessing this NUMA node.",
 687         format_description => "id[-id];...",
 688     },
 689     memory => {
 690         type => "number",
 691         description => "Amount of memory this NUMA node provides.",
 692         optional => 1,
 693     },
 694     hostnodes => {
 695         type => "string",
 696         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 697         description => "Host NUMA nodes to use.",
 698         format_description => "id[-id];...",
 699         optional => 1,
 700     },
 701     policy => {
 702         type => 'string',
 703         enum => [qw(preferred bind interleave)],
 704         description => "NUMA allocation policy.",
 705         optional => 1,
 706     },
 707 };
 708 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 709 my $numadesc = {
 710     optional => 1,
 711     type => 'string', format => $numa_fmt,
 712     description => "NUMA topology.",
 713 };
 714 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 715
 716 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 717     $confdesc->{"numa$i"} = $numadesc;
 718 }
 719
 720 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 721                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 722                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 723 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 724
 725 my $net_fmt_bridge_descr = <<__EOD__;
 726 Bridge to attach the network device to. The Proxmox VE standard bridge
 727 is called 'vmbr0'.
 728
 729 If you do not specify a bridge, we create a kvm user (NATed) network
 730 device, which provides DHCP and DNS services. The following addresses
 731 are used:
 732
 733  10.0.2.2   Gateway
 734  10.0.2.3   DNS Server
 735  10.0.2.4   SMB Server
 736
 737 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 738 __EOD__
 739
 740 my $net_fmt = {
 741     macaddr => {
 742         type => 'string',
 743         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 744         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 745         format_description => "XX:XX:XX:XX:XX:XX",
 746         optional => 1,
 747     },
 748     model => {
 749         type => 'string',
 750         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 751         enum => $nic_model_list,
 752         default_key => 1,
 753     },
 754     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 755     bridge => {
 756         type => 'string',
 757         description => $net_fmt_bridge_descr,
 758         format_description => 'bridge',
 759         optional => 1,
 760     },
 761     queues => {
 762         type => 'integer',
 763         minimum => 0, maximum => 16,
 764         description => 'Number of packet queues to be used on the device.',
 765         optional => 1,
 766     },
 767     rate => {
 768         type => 'number',
 769         minimum => 0,
 770         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 771         optional => 1,
 772     },
 773     tag => {
 774         type => 'integer',
 775         minimum => 1, maximum => 4094,
 776         description => 'VLAN tag to apply to packets on this interface.',
 777         optional => 1,
 778     },
 779     trunks => {
 780         type => 'string',
 781         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 782         description => 'VLAN trunks to pass through this interface.',
 783         format_description => 'vlanid[;vlanid...]',
 784         optional => 1,
 785     },
 786     firewall => {
 787         type => 'boolean',
 788         description => 'Whether this interface should be protected by the firewall.',
 789         optional => 1,
 790     },
 791     link_down => {
 792         type => 'boolean',
 793         description => 'Whether this interface should be disconnected (like pulling the plug).',
 794         optional => 1,
 795     },
 796 };
 797
 798 my $netdesc = {
 799     optional => 1,
 800     type => 'string', format => $net_fmt,
 801     description => "Specify network devices.",
 802 };
 803
 804 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 805
 806 my $ipconfig_fmt = {
 807     ip => {
 808         type => 'string',
 809         format => 'pve-ipv4-config',
 810         format_description => 'IPv4Format/CIDR',
 811         description => 'IPv4 address in CIDR format.',
 812         optional => 1,
 813         default => 'dhcp',
 814     },
 815     gw => {
 816         type => 'string',
 817         format => 'ipv4',
 818         format_description => 'GatewayIPv4',
 819         description => 'Default gateway for IPv4 traffic.',
 820         optional => 1,
 821         requires => 'ip',
 822     },
 823     ip6 => {
 824         type => 'string',
 825         format => 'pve-ipv6-config',
 826         format_description => 'IPv6Format/CIDR',
 827         description => 'IPv6 address in CIDR format.',
 828         optional => 1,
 829         default => 'dhcp',
 830     },
 831     gw6 => {
 832         type => 'string',
 833         format => 'ipv6',
 834         format_description => 'GatewayIPv6',
 835         description => 'Default gateway for IPv6 traffic.',
 836         optional => 1,
 837         requires => 'ip6',
 838     },
 839 };
 840 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 841 my $ipconfigdesc = {
 842     optional => 1,
 843     type => 'string', format => 'pve-qm-ipconfig',
 844     description => <<'EODESCR',
 845 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 846
 847 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 848
 849 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 850 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 851
 852 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 853 EODESCR
 854 };
 855 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 856
 857 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 858     $confdesc->{"net$i"} = $netdesc;
 859     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 860 }
 861
 862 foreach my $key (keys %$confdesc_cloudinit) {
 863     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 864 }
 865
 866 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 867 sub verify_volume_id_or_qm_path {
 868     my ($volid, $noerr) = @_;
 869
 870     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 871         return $volid;
 872     }
 873
 874     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 875     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 876     if ($@) {
 877         return undef if $noerr;
 878         die $@;
 879     }
 880     return $volid;
 881 }
 882
 883 my $drivename_hash;
 884
 885 my %drivedesc_base = (
 886     volume => { alias => 'file' },
 887     file => {
 888         type => 'string',
 889         format => 'pve-volume-id-or-qm-path',
 890         default_key => 1,
 891         format_description => 'volume',
 892         description => "The drive's backing volume.",
 893     },
 894     media => {
 895         type => 'string',
 896         enum => [qw(cdrom disk)],
 897         description => "The drive's media type.",
 898         default => 'disk',
 899         optional => 1
 900     },
 901     cyls => {
 902         type => 'integer',
 903         description => "Force the drive's physical geometry to have a specific cylinder count.",
 904         optional => 1
 905     },
 906     heads => {
 907         type => 'integer',
 908         description => "Force the drive's physical geometry to have a specific head count.",
 909         optional => 1
 910     },
 911     secs => {
 912         type => 'integer',
 913         description => "Force the drive's physical geometry to have a specific sector count.",
 914         optional => 1
 915     },
 916     trans => {
 917         type => 'string',
 918         enum => [qw(none lba auto)],
 919         description => "Force disk geometry bios translation mode.",
 920         optional => 1,
 921     },
 922     snapshot => {
 923         type => 'boolean',
 924         description => "Controls qemu's snapshot mode feature."
 925             . " If activated, changes made to the disk are temporary and will"
 926             . " be discarded when the VM is shutdown.",
 927         optional => 1,
 928     },
 929     cache => {
 930         type => 'string',
 931         enum => [qw(none writethrough writeback unsafe directsync)],
 932         description => "The drive's cache mode",
 933         optional => 1,
 934     },
 935     format => get_standard_option('pve-qm-image-format'),
 936     size => {
 937         type => 'string',
 938         format => 'disk-size',
 939         format_description => 'DiskSize',
 940         description => "Disk size. This is purely informational and has no effect.",
 941         optional => 1,
 942     },
 943     backup => {
 944         type => 'boolean',
 945         description => "Whether the drive should be included when making backups.",
 946         optional => 1,
 947     },
 948     replicate => {
 949         type => 'boolean',
 950         description => 'Whether the drive should considered for replication jobs.',
 951         optional => 1,
 952         default => 1,
 953     },
 954     rerror => {
 955         type => 'string',
 956         enum => [qw(ignore report stop)],
 957         description => 'Read error action.',
 958         optional => 1,
 959     },
 960     werror => {
 961         type => 'string',
 962         enum => [qw(enospc ignore report stop)],
 963         description => 'Write error action.',
 964         optional => 1,
 965     },
 966     aio => {
 967         type => 'string',
 968         enum => [qw(native threads)],
 969         description => 'AIO type to use.',
 970         optional => 1,
 971     },
 972     discard => {
 973         type => 'string',
 974         enum => [qw(ignore on)],
 975         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 976         optional => 1,
 977     },
 978     detect_zeroes => {
 979         type => 'boolean',
 980         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 981         optional => 1,
 982     },
 983     serial => {
 984         type => 'string',
 985         format => 'urlencoded',
 986         format_description => 'serial',
 987         maxLength => 20*3, # *3 since it's %xx url enoded
 988         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 989         optional => 1,
 990     },
 991     shared => {
 992         type => 'boolean',
 993         description => 'Mark this locally-managed volume as available on all nodes',
 994         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 995         optional => 1,
 996         default => 0,
 997     }
 998 );
 999
1000 my %iothread_fmt = ( iothread => {
1001         type => 'boolean',
1002         description => "Whether to use iothreads for this drive",
1003         optional => 1,
1004 });
1005
1006 my %model_fmt = (
1007     model => {
1008         type => 'string',
1009         format => 'urlencoded',
1010         format_description => 'model',
1011         maxLength => 40*3, # *3 since it's %xx url enoded
1012         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1013         optional => 1,
1014     },
1015 );
1016
1017 my %queues_fmt = (
1018     queues => {
1019         type => 'integer',
1020         description => "Number of queues.",
1021         minimum => 2,
1022         optional => 1
1023     }
1024 );
1025
1026 my %scsiblock_fmt = (
1027     scsiblock => {
1028         type => 'boolean',
1029         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1030         optional => 1,
1031         default => 0,
1032     },
1033 );
1034
1035 my %ssd_fmt = (
1036     ssd => {
1037         type => 'boolean',
1038         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1039         optional => 1,
1040     },
1041 );
1042
1043 my $add_throttle_desc = sub {
1044     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1045     my $d = {
1046         type => $type,
1047         format_description => $unit,
1048         description => "Maximum $what in $longunit.",
1049         optional => 1,
1050     };
1051     $d->{minimum} = $minimum if defined($minimum);
1052     $drivedesc_base{$key} = $d;
1053 };
1054 # throughput: (leaky bucket)
1055 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1056 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1057 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1058 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1060 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1061 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1062 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1063 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1064
1065 # pools: (pool of IO before throttling starts taking effect)
1066 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1067 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1068 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1069 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1070 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1071 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1072
1073 # burst lengths
1074 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1075 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1076 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1078 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1079 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1080
1081 # legacy support
1082 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1083 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1084 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1085 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1086
1087 my $ide_fmt = {
1088     %drivedesc_base,
1089     %model_fmt,
1090     %ssd_fmt,
1091 };
1092 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1093
1094 my $idedesc = {
1095     optional => 1,
1096     type => 'string', format => $ide_fmt,
1097     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1098 };
1099 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1100
1101 my $scsi_fmt = {
1102     %drivedesc_base,
1103     %iothread_fmt,
1104     %queues_fmt,
1105     %scsiblock_fmt,
1106     %ssd_fmt,
1107 };
1108 my $scsidesc = {
1109     optional => 1,
1110     type => 'string', format => $scsi_fmt,
1111     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1112 };
1113 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1114
1115 my $sata_fmt = {
1116     %drivedesc_base,
1117     %ssd_fmt,
1118 };
1119 my $satadesc = {
1120     optional => 1,
1121     type => 'string', format => $sata_fmt,
1122     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1123 };
1124 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1125
1126 my $virtio_fmt = {
1127     %drivedesc_base,
1128     %iothread_fmt,
1129 };
1130 my $virtiodesc = {
1131     optional => 1,
1132     type => 'string', format => $virtio_fmt,
1133     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1134 };
1135 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1136
1137 my $alldrive_fmt = {
1138     %drivedesc_base,
1139     %iothread_fmt,
1140     %model_fmt,
1141     %queues_fmt,
1142     %scsiblock_fmt,
1143     %ssd_fmt,
1144 };
1145
1146 my $efidisk_fmt = {
1147     volume => { alias => 'file' },
1148     file => {
1149         type => 'string',
1150         format => 'pve-volume-id-or-qm-path',
1151         default_key => 1,
1152         format_description => 'volume',
1153         description => "The drive's backing volume.",
1154     },
1155     format => get_standard_option('pve-qm-image-format'),
1156     size => {
1157         type => 'string',
1158         format => 'disk-size',
1159         format_description => 'DiskSize',
1160         description => "Disk size. This is purely informational and has no effect.",
1161         optional => 1,
1162     },
1163 };
1164
1165 my $efidisk_desc = {
1166     optional => 1,
1167     type => 'string', format => $efidisk_fmt,
1168     description => "Configure a Disk for storing EFI vars",
1169 };
1170
1171 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1172
1173 my $usb_fmt = {
1174     host => {
1175         default_key => 1,
1176         type => 'string', format => 'pve-qm-usb-device',
1177         format_description => 'HOSTUSBDEVICE|spice',
1178         description => <<EODESCR,
1179 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1180
1181  'bus-port(.port)*' (decimal numbers) or
1182  'vendor_id:product_id' (hexadeciaml numbers) or
1183  'spice'
1184
1185 You can use the 'lsusb -t' command to list existing usb devices.
1186
1187 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1188
1189 The value 'spice' can be used to add a usb redirection devices for spice.
1190 EODESCR
1191     },
1192     usb3 => {
1193         optional => 1,
1194         type => 'boolean',
1195         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1196         default => 0,
1197     },
1198 };
1199
1200 my $usbdesc = {
1201     optional => 1,
1202     type => 'string', format => $usb_fmt,
1203     description => "Configure an USB device (n is 0 to 4).",
1204 };
1205 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1206
1207 # NOTE: the match-groups of this regex are used in parse_hostpci
1208 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1209 my $hostpci_fmt = {
1210     host => {
1211         default_key => 1,
1212         type => 'string',
1213         pattern => qr/$PCIRE(;$PCIRE)*/,
1214         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1215         description => <<EODESCR,
1216 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1217 of PCI virtual functions of the host. HOSTPCIID syntax is:
1218
1219 'bus:dev.func' (hexadecimal numbers)
1220
1221 You can us the 'lspci' command to list existing PCI devices.
1222 EODESCR
1223     },
1224     rombar => {
1225         type => 'boolean',
1226         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1227         optional => 1,
1228         default => 1,
1229     },
1230     romfile => {
1231         type => 'string',
1232         pattern => '[^,;]+',
1233         format_description => 'string',
1234         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1235         optional => 1,
1236     },
1237     pcie => {
1238         type => 'boolean',
1239         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1240         optional => 1,
1241         default => 0,
1242     },
1243     'x-vga' => {
1244         type => 'boolean',
1245         description =>  "Enable vfio-vga device support.",
1246         optional => 1,
1247         default => 0,
1248     },
1249 };
1250 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1251
1252 my $hostpcidesc = {
1253         optional => 1,
1254         type => 'string', format => 'pve-qm-hostpci',
1255         description => "Map host PCI devices into guest.",
1256         verbose_description =>  <<EODESCR,
1257 Map host PCI devices into guest.
1258
1259 NOTE: This option allows direct access to host hardware. So it is no longer
1260 possible to migrate such machines - use with special care.
1261
1262 CAUTION: Experimental! User reported problems with this option.
1263 EODESCR
1264 };
1265 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1266
1267 my $serialdesc = {
1268         optional => 1,
1269         type => 'string',
1270         pattern => '(/dev/.+|socket)',
1271         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1272         verbose_description =>  <<EODESCR,
1273 Create a serial device inside the VM (n is 0 to 3), and pass through a
1274 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1275 host side (use 'qm terminal' to open a terminal connection).
1276
1277 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1278
1279 CAUTION: Experimental! User reported problems with this option.
1280 EODESCR
1281 };
1282
1283 my $paralleldesc= {
1284         optional => 1,
1285         type => 'string',
1286         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1287         description =>  "Map host parallel devices (n is 0 to 2).",
1288         verbose_description =>  <<EODESCR,
1289 Map host parallel devices (n is 0 to 2).
1290
1291 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1292
1293 CAUTION: Experimental! User reported problems with this option.
1294 EODESCR
1295 };
1296
1297 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1298     $confdesc->{"parallel$i"} = $paralleldesc;
1299 }
1300
1301 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1302     $confdesc->{"serial$i"} = $serialdesc;
1303 }
1304
1305 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1306     $confdesc->{"hostpci$i"} = $hostpcidesc;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1310     $drivename_hash->{"ide$i"} = 1;
1311     $confdesc->{"ide$i"} = $idedesc;
1312 }
1313
1314 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1315     $drivename_hash->{"sata$i"} = 1;
1316     $confdesc->{"sata$i"} = $satadesc;
1317 }
1318
1319 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1320     $drivename_hash->{"scsi$i"} = 1;
1321     $confdesc->{"scsi$i"} = $scsidesc ;
1322 }
1323
1324 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1325     $drivename_hash->{"virtio$i"} = 1;
1326     $confdesc->{"virtio$i"} = $virtiodesc;
1327 }
1328
1329 $drivename_hash->{efidisk0} = 1;
1330 $confdesc->{efidisk0} = $efidisk_desc;
1331
1332 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1333     $confdesc->{"usb$i"} = $usbdesc;
1334 }
1335
1336 my $unuseddesc = {
1337     optional => 1,
1338     type => 'string', format => 'pve-volume-id',
1339     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1340 };
1341
1342 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1343     $confdesc->{"unused$i"} = $unuseddesc;
1344 }
1345
1346 my $kvm_api_version = 0;
1347
1348 sub kvm_version {
1349     return $kvm_api_version if $kvm_api_version;
1350
1351     open my $fh, '<', '/dev/kvm'
1352         or return undef;
1353
1354     # 0xae00 => KVM_GET_API_VERSION
1355     $kvm_api_version = ioctl($fh, 0xae00, 0);
1356
1357     return $kvm_api_version;
1358 }
1359
1360 my $kvm_user_version;
1361
1362 sub kvm_user_version {
1363
1364     return $kvm_user_version if $kvm_user_version;
1365
1366     $kvm_user_version = 'unknown';
1367
1368     my $code = sub {
1369         my $line = shift;
1370         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1371             $kvm_user_version = $2;
1372         }
1373     };
1374
1375     eval { run_command("kvm -version", outfunc => $code); };
1376     warn $@ if $@;
1377
1378     return $kvm_user_version;
1379
1380 }
1381
1382 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1383
1384 sub valid_drive_names {
1385     # order is important - used to autoselect boot disk
1386     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1387             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1388             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1389             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1390             'efidisk0');
1391 }
1392
1393 sub is_valid_drivename {
1394     my $dev = shift;
1395
1396     return defined($drivename_hash->{$dev});
1397 }
1398
1399 sub option_exists {
1400     my $key = shift;
1401     return defined($confdesc->{$key});
1402 }
1403
1404 sub nic_models {
1405     return $nic_model_list;
1406 }
1407
1408 sub os_list_description {
1409
1410     return {
1411         other => 'Other',
1412         wxp => 'Windows XP',
1413         w2k => 'Windows 2000',
1414         w2k3 =>, 'Windows 2003',
1415         w2k8 => 'Windows 2008',
1416         wvista => 'Windows Vista',
1417         win7 => 'Windows 7',
1418         win8 => 'Windows 8/2012',
1419         win10 => 'Windows 10/2016',
1420         l24 => 'Linux 2.4',
1421         l26 => 'Linux 2.6',
1422     };
1423 }
1424
1425 my $cdrom_path;
1426
1427 sub get_cdrom_path {
1428
1429     return  $cdrom_path if $cdrom_path;
1430
1431     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1432     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1433     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1434 }
1435
1436 sub get_iso_path {
1437     my ($storecfg, $vmid, $cdrom) = @_;
1438
1439     if ($cdrom eq 'cdrom') {
1440         return get_cdrom_path();
1441     } elsif ($cdrom eq 'none') {
1442         return '';
1443     } elsif ($cdrom =~ m|^/|) {
1444         return $cdrom;
1445     } else {
1446         return PVE::Storage::path($storecfg, $cdrom);
1447     }
1448 }
1449
1450 # try to convert old style file names to volume IDs
1451 sub filename_to_volume_id {
1452     my ($vmid, $file, $media) = @_;
1453
1454      if (!($file eq 'none' || $file eq 'cdrom' ||
1455           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1456
1457         return undef if $file =~ m|/|;
1458
1459         if ($media && $media eq 'cdrom') {
1460             $file = "local:iso/$file";
1461         } else {
1462             $file = "local:$vmid/$file";
1463         }
1464     }
1465
1466     return $file;
1467 }
1468
1469 sub verify_media_type {
1470     my ($opt, $vtype, $media) = @_;
1471
1472     return if !$media;
1473
1474     my $etype;
1475     if ($media eq 'disk') {
1476         $etype = 'images';
1477     } elsif ($media eq 'cdrom') {
1478         $etype = 'iso';
1479     } else {
1480         die "internal error";
1481     }
1482
1483     return if ($vtype eq $etype);
1484
1485     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1486 }
1487
1488 sub cleanup_drive_path {
1489     my ($opt, $storecfg, $drive) = @_;
1490
1491     # try to convert filesystem paths to volume IDs
1492
1493     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1494         ($drive->{file} !~ m|^/dev/.+|) &&
1495         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1496         ($drive->{file} !~ m/^\d+$/)) {
1497         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1498         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1499         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1500         verify_media_type($opt, $vtype, $drive->{media});
1501         $drive->{file} = $volid;
1502     }
1503
1504     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1505 }
1506
1507 sub parse_hotplug_features {
1508     my ($data) = @_;
1509
1510     my $res = {};
1511
1512     return $res if $data eq '0';
1513
1514     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1515
1516     foreach my $feature (PVE::Tools::split_list($data)) {
1517         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1518             $res->{$1} = 1;
1519         } else {
1520             die "invalid hotplug feature '$feature'\n";
1521         }
1522     }
1523     return $res;
1524 }
1525
1526 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1527 sub pve_verify_hotplug_features {
1528     my ($value, $noerr) = @_;
1529
1530     return $value if parse_hotplug_features($value);
1531
1532     return undef if $noerr;
1533
1534     die "unable to parse hotplug option\n";
1535 }
1536
1537 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1538 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1539 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1540 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1541 #        [,iothread=on][,serial=serial][,model=model]
1542
1543 sub parse_drive {
1544     my ($key, $data) = @_;
1545
1546     my ($interface, $index);
1547
1548     if ($key =~ m/^([^\d]+)(\d+)$/) {
1549         $interface = $1;
1550         $index = $2;
1551     } else {
1552         return undef;
1553     }
1554
1555     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1556                                      : $confdesc->{$key}->{format};
1557     if (!$desc) {
1558         warn "invalid drive key: $key\n";
1559         return undef;
1560     }
1561     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1562     return undef if !$res;
1563     $res->{interface} = $interface;
1564     $res->{index} = $index;
1565
1566     my $error = 0;
1567     foreach my $opt (qw(bps bps_rd bps_wr)) {
1568         if (my $bps = defined(delete $res->{$opt})) {
1569             if (defined($res->{"m$opt"})) {
1570                 warn "both $opt and m$opt specified\n";
1571                 ++$error;
1572                 next;
1573             }
1574             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1575         }
1576     }
1577
1578     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1579     for my $requirement (
1580         [mbps_max => 'mbps'],
1581         [mbps_rd_max => 'mbps_rd'],
1582         [mbps_wr_max => 'mbps_wr'],
1583         [miops_max => 'miops'],
1584         [miops_rd_max => 'miops_rd'],
1585         [miops_wr_max => 'miops_wr'],
1586         [bps_max_length => 'mbps_max'],
1587         [bps_rd_max_length => 'mbps_rd_max'],
1588         [bps_wr_max_length => 'mbps_wr_max'],
1589         [iops_max_length => 'iops_max'],
1590         [iops_rd_max_length => 'iops_rd_max'],
1591         [iops_wr_max_length => 'iops_wr_max']) {
1592         my ($option, $requires) = @$requirement;
1593         if ($res->{$option} && !$res->{$requires}) {
1594             warn "$option requires $requires\n";
1595             ++$error;
1596         }
1597     }
1598
1599     return undef if $error;
1600
1601     return undef if $res->{mbps_rd} && $res->{mbps};
1602     return undef if $res->{mbps_wr} && $res->{mbps};
1603     return undef if $res->{iops_rd} && $res->{iops};
1604     return undef if $res->{iops_wr} && $res->{iops};
1605
1606     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1607         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1608         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1609         return undef if $res->{interface} eq 'virtio';
1610     }
1611
1612     if (my $size = $res->{size}) {
1613         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1614     }
1615
1616     return $res;
1617 }
1618
1619 sub print_drive {
1620     my ($vmid, $drive) = @_;
1621     my $data = { %$drive };
1622     delete $data->{$_} for qw(index interface);
1623     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1624 }
1625
1626 sub scsi_inquiry {
1627     my($fh, $noerr) = @_;
1628
1629     my $SG_IO = 0x2285;
1630     my $SG_GET_VERSION_NUM = 0x2282;
1631
1632     my $versionbuf = "\x00" x 8;
1633     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1634     if (!$ret) {
1635         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1636         return undef;
1637     }
1638     my $version = unpack("I", $versionbuf);
1639     if ($version < 30000) {
1640         die "scsi generic interface too old\n"  if !$noerr;
1641         return undef;
1642     }
1643
1644     my $buf = "\x00" x 36;
1645     my $sensebuf = "\x00" x 8;
1646     my $cmd = pack("C x3 C x1", 0x12, 36);
1647
1648     # see /usr/include/scsi/sg.h
1649     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1650
1651     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1652                       length($sensebuf), 0, length($buf), $buf,
1653                       $cmd, $sensebuf, 6000);
1654
1655     $ret = ioctl($fh, $SG_IO, $packet);
1656     if (!$ret) {
1657         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1658         return undef;
1659     }
1660
1661     my @res = unpack($sg_io_hdr_t, $packet);
1662     if ($res[17] || $res[18]) {
1663         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1664         return undef;
1665     }
1666
1667     my $res = {};
1668     (my $byte0, my $byte1, $res->{vendor},
1669      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1670
1671     $res->{removable} = $byte1 & 128 ? 1 : 0;
1672     $res->{type} = $byte0 & 31;
1673
1674     return $res;
1675 }
1676
1677 sub path_is_scsi {
1678     my ($path) = @_;
1679
1680     my $fh = IO::File->new("+<$path") || return undef;
1681     my $res = scsi_inquiry($fh, 1);
1682     close($fh);
1683
1684     return $res;
1685 }
1686
1687 sub machine_type_is_q35 {
1688     my ($conf) = @_;
1689
1690     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1691 }
1692
1693 sub print_tabletdevice_full {
1694     my ($conf, $arch) = @_;
1695
1696     my $q35 = machine_type_is_q35($conf);
1697
1698     # we use uhci for old VMs because tablet driver was buggy in older qemu
1699     my $usbbus;
1700     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1701         $usbbus = 'ehci';
1702     } else {
1703         $usbbus = 'uhci';
1704     }
1705
1706     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1707 }
1708
1709 sub print_keyboarddevice_full {
1710     my ($conf, $arch, $machine) = @_;
1711
1712     return undef if $arch ne 'aarch64';
1713
1714     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1715 }
1716
1717 sub print_drivedevice_full {
1718     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1719
1720     my $device = '';
1721     my $maxdev = 0;
1722
1723     if ($drive->{interface} eq 'virtio') {
1724         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1725         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1726         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1727     } elsif ($drive->{interface} eq 'scsi') {
1728
1729         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1730         my $unit = $drive->{index} % $maxdev;
1731         my $devicetype = 'hd';
1732         my $path = '';
1733         if (drive_is_cdrom($drive)) {
1734             $devicetype = 'cd';
1735         } else {
1736             if ($drive->{file} =~ m|^/|) {
1737                 $path = $drive->{file};
1738                 if (my $info = path_is_scsi($path)) {
1739                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1740                         $devicetype = 'block';
1741                     } elsif ($info->{type} == 1) { # tape
1742                         $devicetype = 'generic';
1743                     }
1744                 }
1745             } else {
1746                  $path = PVE::Storage::path($storecfg, $drive->{file});
1747             }
1748
1749             if($path =~ m/^iscsi\:\/\//){
1750                 $devicetype = 'generic';
1751             }
1752         }
1753
1754         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1755             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1756         } else {
1757             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1758         }
1759
1760         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1761             $device .= ",rotation_rate=1";
1762         }
1763
1764     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1765         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1766         my $controller = int($drive->{index} / $maxdev);
1767         my $unit = $drive->{index} % $maxdev;
1768         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1769
1770         $device = "ide-$devicetype";
1771         if ($drive->{interface} eq 'ide') {
1772             $device .= ",bus=ide.$controller,unit=$unit";
1773         } else {
1774             $device .= ",bus=ahci$controller.$unit";
1775         }
1776         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1777
1778         if ($devicetype eq 'hd') {
1779             if (my $model = $drive->{model}) {
1780                 $model = URI::Escape::uri_unescape($model);
1781                 $device .= ",model=$model";
1782             }
1783             if ($drive->{ssd}) {
1784                 $device .= ",rotation_rate=1";
1785             }
1786         }
1787     } elsif ($drive->{interface} eq 'usb') {
1788         die "implement me";
1789         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1790     } else {
1791         die "unsupported interface type";
1792     }
1793
1794     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1795
1796     if (my $serial = $drive->{serial}) {
1797         $serial = URI::Escape::uri_unescape($serial);
1798         $device .= ",serial=$serial";
1799     }
1800
1801
1802     return $device;
1803 }
1804
1805 sub get_initiator_name {
1806     my $initiator;
1807
1808     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1809     while (defined(my $line = <$fh>)) {
1810         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1811         $initiator = $1;
1812         last;
1813     }
1814     $fh->close();
1815
1816     return $initiator;
1817 }
1818
1819 sub print_drive_full {
1820     my ($storecfg, $vmid, $drive) = @_;
1821
1822     my $path;
1823     my $volid = $drive->{file};
1824     my $format;
1825
1826     if (drive_is_cdrom($drive)) {
1827         $path = get_iso_path($storecfg, $vmid, $volid);
1828     } else {
1829         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1830         if ($storeid) {
1831             $path = PVE::Storage::path($storecfg, $volid);
1832             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1833             $format = qemu_img_format($scfg, $volname);
1834         } else {
1835             $path = $volid;
1836             $format = "raw";
1837         }
1838    }
1839
1840     my $opts = '';
1841     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1842     foreach my $o (@qemu_drive_options) {
1843         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1844     }
1845
1846     # snapshot only accepts on|off
1847     if (defined($drive->{snapshot})) {
1848         my $v = $drive->{snapshot} ? 'on' : 'off';
1849         $opts .= ",snapshot=$v";
1850     }
1851
1852     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1853         my ($dir, $qmpname) = @$type;
1854         if (my $v = $drive->{"mbps$dir"}) {
1855             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1856         }
1857         if (my $v = $drive->{"mbps${dir}_max"}) {
1858             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1859         }
1860         if (my $v = $drive->{"bps${dir}_max_length"}) {
1861             $opts .= ",throttling.bps$qmpname-max-length=$v";
1862         }
1863         if (my $v = $drive->{"iops${dir}"}) {
1864             $opts .= ",throttling.iops$qmpname=$v";
1865         }
1866         if (my $v = $drive->{"iops${dir}_max"}) {
1867             $opts .= ",throttling.iops$qmpname-max=$v";
1868         }
1869         if (my $v = $drive->{"iops${dir}_max_length"}) {
1870             $opts .= ",throttling.iops$qmpname-max-length=$v";
1871         }
1872     }
1873
1874     $opts .= ",format=$format" if $format && !$drive->{format};
1875
1876     my $cache_direct = 0;
1877
1878     if (my $cache = $drive->{cache}) {
1879         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1880     } elsif (!drive_is_cdrom($drive)) {
1881         $opts .= ",cache=none";
1882         $cache_direct = 1;
1883     }
1884
1885     # aio native works only with O_DIRECT
1886     if (!$drive->{aio}) {
1887         if($cache_direct) {
1888             $opts .= ",aio=native";
1889         } else {
1890             $opts .= ",aio=threads";
1891         }
1892     }
1893
1894     if (!drive_is_cdrom($drive)) {
1895         my $detectzeroes;
1896         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1897             $detectzeroes = 'off';
1898         } elsif ($drive->{discard}) {
1899             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1900         } else {
1901             # This used to be our default with discard not being specified:
1902             $detectzeroes = 'on';
1903         }
1904         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1905     }
1906
1907     my $pathinfo = $path ? "file=$path," : '';
1908
1909     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1910 }
1911
1912 sub print_netdevice_full {
1913     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1914
1915     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1916
1917     my $device = $net->{model};
1918     if ($net->{model} eq 'virtio') {
1919          $device = 'virtio-net-pci';
1920      };
1921
1922     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1923     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1924     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1925         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1926         my $vectors = $net->{queues} * 2 + 2;
1927         $tmpstr .= ",vectors=$vectors,mq=on";
1928     }
1929     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1930
1931     if ($use_old_bios_files) {
1932         my $romfile;
1933         if ($device eq 'virtio-net-pci') {
1934             $romfile = 'pxe-virtio.rom';
1935         } elsif ($device eq 'e1000') {
1936             $romfile = 'pxe-e1000.rom';
1937         } elsif ($device eq 'ne2k') {
1938             $romfile = 'pxe-ne2k_pci.rom';
1939         } elsif ($device eq 'pcnet') {
1940             $romfile = 'pxe-pcnet.rom';
1941         } elsif ($device eq 'rtl8139') {
1942             $romfile = 'pxe-rtl8139.rom';
1943         }
1944         $tmpstr .= ",romfile=$romfile" if $romfile;
1945     }
1946
1947     return $tmpstr;
1948 }
1949
1950 sub print_netdev_full {
1951     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1952
1953     my $i = '';
1954     if ($netid =~ m/^net(\d+)$/) {
1955         $i = int($1);
1956     }
1957
1958     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1959
1960     my $ifname = "tap${vmid}i$i";
1961
1962     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1963     die "interface name '$ifname' is too long (max 15 character)\n"
1964         if length($ifname) >= 16;
1965
1966     my $vhostparam = '';
1967     if (is_native($arch)) {
1968         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1969     }
1970
1971     my $vmname = $conf->{name} || "vm$vmid";
1972
1973     my $netdev = "";
1974     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1975
1976     if ($net->{bridge}) {
1977         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1978     } else {
1979         $netdev = "type=user,id=$netid,hostname=$vmname";
1980     }
1981
1982     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1983
1984     return $netdev;
1985 }
1986
1987
1988 sub print_cpu_device {
1989     my ($conf, $id) = @_;
1990
1991     my $kvm = $conf->{kvm} // 1;
1992     my $cpu = $kvm ? "kvm64" : "qemu64";
1993     if (my $cputype = $conf->{cpu}) {
1994         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1995             or die "Cannot parse cpu description: $cputype\n";
1996         $cpu = $cpuconf->{cputype};
1997     }
1998
1999     my $cores = $conf->{cores} || 1;
2000
2001     my $current_core = ($id - 1) % $cores;
2002     my $current_socket = int(($id - 1 - $current_core)/$cores);
2003
2004     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2005 }
2006
2007 my $vga_map = {
2008     'cirrus' => 'cirrus-vga',
2009     'std' => 'VGA',
2010     'vmware' => 'vmware-svga',
2011     'virtio' => 'virtio-vga',
2012 };
2013
2014 sub print_vga_device {
2015     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2016
2017     my $type = $vga_map->{$vga->{type}};
2018     if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
2019         $type = 'virtio-gpu';
2020     }
2021     my $vgamem_mb = $vga->{memory};
2022     if ($qxlnum) {
2023         $type = $id ? 'qxl' : 'qxl-vga';
2024     }
2025     die "no devicetype for $vga->{type}\n" if !$type;
2026
2027     my $memory = "";
2028     if ($vgamem_mb) {
2029         if ($vga->{type} eq 'virtio') {
2030             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2031             $memory = ",max_hostmem=$bytes";
2032         } elsif ($qxlnum) {
2033             # from https://www.spice-space.org/multiple-monitors.html
2034             $memory = ",vgamem_mb=$vga->{memory}";
2035             my $ram = $vgamem_mb * 4;
2036             my $vram = $vgamem_mb * 2;
2037             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2038         } else {
2039             $memory = ",vgamem_mb=$vga->{memory}";
2040         }
2041     } elsif ($qxlnum && $id) {
2042         $memory = ",ram_size=67108864,vram_size=33554432";
2043     }
2044
2045     my $q35 = machine_type_is_q35($conf);
2046     my $vgaid = "vga" . ($id // '');
2047     my $pciaddr;
2048
2049     if ($q35 && $vgaid eq 'vga') {
2050         # the first display uses pcie.0 bus on q35 machines
2051         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2052     } else {
2053         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2054     }
2055
2056     return "$type,id=${vgaid}${memory}${pciaddr}";
2057 }
2058
2059 sub drive_is_cloudinit {
2060     my ($drive) = @_;
2061     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2062 }
2063
2064 sub drive_is_cdrom {
2065     my ($drive, $exclude_cloudinit) = @_;
2066
2067     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2068
2069     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2070
2071 }
2072
2073 sub parse_number_sets {
2074     my ($set) = @_;
2075     my $res = [];
2076     foreach my $part (split(/;/, $set)) {
2077         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2078             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2079             push @$res, [ $1, $2 ];
2080         } else {
2081             die "invalid range: $part\n";
2082         }
2083     }
2084     return $res;
2085 }
2086
2087 sub parse_numa {
2088     my ($data) = @_;
2089
2090     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2091     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2092     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2093     return $res;
2094 }
2095
2096 sub parse_hostpci {
2097     my ($value) = @_;
2098
2099     return undef if !$value;
2100
2101     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2102
2103     my @idlist = split(/;/, $res->{host});
2104     delete $res->{host};
2105     foreach my $id (@idlist) {
2106         if ($id =~ /^$PCIRE$/) {
2107             if (defined($2)) {
2108                 push @{$res->{pciid}}, { id => $1, function => $2 };
2109             } else {
2110                 my $pcidevices = PVE::SysFSTools::lspci($1);
2111                 $res->{pciid} = $pcidevices->{$1};
2112             }
2113         } else {
2114             # should have been caught by parse_property_string already
2115             die "failed to parse PCI id: $id\n";
2116         }
2117     }
2118     return $res;
2119 }
2120
2121 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2122 sub parse_net {
2123     my ($data) = @_;
2124
2125     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2126     if ($@) {
2127         warn $@;
2128         return undef;
2129     }
2130     if (!defined($res->{macaddr})) {
2131         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2132         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2133     }
2134     return $res;
2135 }
2136
2137 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2138 sub parse_ipconfig {
2139     my ($data) = @_;
2140
2141     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2142     if ($@) {
2143         warn $@;
2144         return undef;
2145     }
2146
2147     if ($res->{gw} && !$res->{ip}) {
2148         warn 'gateway specified without specifying an IP address';
2149         return undef;
2150     }
2151     if ($res->{gw6} && !$res->{ip6}) {
2152         warn 'IPv6 gateway specified without specifying an IPv6 address';
2153         return undef;
2154     }
2155     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2156         warn 'gateway specified together with DHCP';
2157         return undef;
2158     }
2159     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2160         # gw6 + auto/dhcp
2161         warn "IPv6 gateway specified together with $res->{ip6} address";
2162         return undef;
2163     }
2164
2165     if (!$res->{ip} && !$res->{ip6}) {
2166         return { ip => 'dhcp', ip6 => 'dhcp' };
2167     }
2168
2169     return $res;
2170 }
2171
2172 sub print_net {
2173     my $net = shift;
2174
2175     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2176 }
2177
2178 sub add_random_macs {
2179     my ($settings) = @_;
2180
2181     foreach my $opt (keys %$settings) {
2182         next if $opt !~ m/^net(\d+)$/;
2183         my $net = parse_net($settings->{$opt});
2184         next if !$net;
2185         $settings->{$opt} = print_net($net);
2186     }
2187 }
2188
2189 sub vm_is_volid_owner {
2190     my ($storecfg, $vmid, $volid) = @_;
2191
2192     if ($volid !~  m|^/|) {
2193         my ($path, $owner);
2194         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2195         if ($owner && ($owner == $vmid)) {
2196             return 1;
2197         }
2198     }
2199
2200     return undef;
2201 }
2202
2203 sub split_flagged_list {
2204     my $text = shift || '';
2205     $text =~ s/[,;]/ /g;
2206     $text =~ s/^\s+//;
2207     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2208 }
2209
2210 sub join_flagged_list {
2211     my ($how, $lst) = @_;
2212     join $how, map { $lst->{$_} . $_ } keys %$lst;
2213 }
2214
2215 sub vmconfig_delete_pending_option {
2216     my ($conf, $key, $force) = @_;
2217
2218     delete $conf->{pending}->{$key};
2219     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2220     $pending_delete_hash->{$key} = $force ? '!' : '';
2221     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2222 }
2223
2224 sub vmconfig_undelete_pending_option {
2225     my ($conf, $key) = @_;
2226
2227     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2228     delete $pending_delete_hash->{$key};
2229
2230     if (%$pending_delete_hash) {
2231         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2232     } else {
2233         delete $conf->{pending}->{delete};
2234     }
2235 }
2236
2237 sub vmconfig_register_unused_drive {
2238     my ($storecfg, $vmid, $conf, $drive) = @_;
2239
2240     if (drive_is_cloudinit($drive)) {
2241         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2242         warn $@ if $@;
2243     } elsif (!drive_is_cdrom($drive)) {
2244         my $volid = $drive->{file};
2245         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2246             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2247         }
2248     }
2249 }
2250
2251 sub vmconfig_cleanup_pending {
2252     my ($conf) = @_;
2253
2254     # remove pending changes when nothing changed
2255     my $changes;
2256     foreach my $opt (keys %{$conf->{pending}}) {
2257         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2258             $changes = 1;
2259             delete $conf->{pending}->{$opt};
2260         }
2261     }
2262
2263     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2264     my $pending_delete_hash = {};
2265     while (my ($opt, $force) = each %$current_delete_hash) {
2266         if (defined($conf->{$opt})) {
2267             $pending_delete_hash->{$opt} = $force;
2268         } else {
2269             $changes = 1;
2270         }
2271     }
2272
2273     if (%$pending_delete_hash) {
2274         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2275     } else {
2276         delete $conf->{pending}->{delete};
2277     }
2278
2279     return $changes;
2280 }
2281
2282 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2283 my $smbios1_fmt = {
2284     uuid => {
2285         type => 'string',
2286         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2287         format_description => 'UUID',
2288         description => "Set SMBIOS1 UUID.",
2289         optional => 1,
2290     },
2291     version => {
2292         type => 'string',
2293         pattern => '\S+',
2294         format_description => 'string',
2295         description => "Set SMBIOS1 version.",
2296         optional => 1,
2297     },
2298     serial => {
2299         type => 'string',
2300         pattern => '\S+',
2301         format_description => 'string',
2302         description => "Set SMBIOS1 serial number.",
2303         optional => 1,
2304     },
2305     manufacturer => {
2306         type => 'string',
2307         pattern => '\S+',
2308         format_description => 'string',
2309         description => "Set SMBIOS1 manufacturer.",
2310         optional => 1,
2311     },
2312     product => {
2313         type => 'string',
2314         pattern => '\S+',
2315         format_description => 'string',
2316         description => "Set SMBIOS1 product ID.",
2317         optional => 1,
2318     },
2319     sku => {
2320         type => 'string',
2321         pattern => '\S+',
2322         format_description => 'string',
2323         description => "Set SMBIOS1 SKU string.",
2324         optional => 1,
2325     },
2326     family => {
2327         type => 'string',
2328         pattern => '\S+',
2329         format_description => 'string',
2330         description => "Set SMBIOS1 family string.",
2331         optional => 1,
2332     },
2333 };
2334
2335 sub parse_smbios1 {
2336     my ($data) = @_;
2337
2338     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2339     warn $@ if $@;
2340     return $res;
2341 }
2342
2343 sub print_smbios1 {
2344     my ($smbios1) = @_;
2345     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2346 }
2347
2348 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2349
2350 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2351 sub verify_bootdisk {
2352     my ($value, $noerr) = @_;
2353
2354     return $value if is_valid_drivename($value);
2355
2356     return undef if $noerr;
2357
2358     die "invalid boot disk '$value'\n";
2359 }
2360
2361 sub parse_watchdog {
2362     my ($value) = @_;
2363
2364     return undef if !$value;
2365
2366     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2367     warn $@ if $@;
2368     return $res;
2369 }
2370
2371 sub parse_guest_agent {
2372     my ($value) = @_;
2373
2374     return {} if !defined($value->{agent});
2375
2376     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2377     warn $@ if $@;
2378
2379     # if the agent is disabled ignore the other potentially set properties
2380     return {} if !$res->{enabled};
2381     return $res;
2382 }
2383
2384 sub parse_vga {
2385     my ($value) = @_;
2386
2387     return {} if !$value;
2388     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2389     warn $@ if $@;
2390     return $res;
2391 }
2392
2393 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2394 sub verify_usb_device {
2395     my ($value, $noerr) = @_;
2396
2397     return $value if parse_usb_device($value);
2398
2399     return undef if $noerr;
2400
2401     die "unable to parse usb device\n";
2402 }
2403
2404 # add JSON properties for create and set function
2405 sub json_config_properties {
2406     my $prop = shift;
2407
2408     foreach my $opt (keys %$confdesc) {
2409         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2410         $prop->{$opt} = $confdesc->{$opt};
2411     }
2412
2413     return $prop;
2414 }
2415
2416 # return copy of $confdesc_cloudinit to generate documentation
2417 sub cloudinit_config_properties {
2418
2419     return dclone($confdesc_cloudinit);
2420 }
2421
2422 sub check_type {
2423     my ($key, $value) = @_;
2424
2425     die "unknown setting '$key'\n" if !$confdesc->{$key};
2426
2427     my $type = $confdesc->{$key}->{type};
2428
2429     if (!defined($value)) {
2430         die "got undefined value\n";
2431     }
2432
2433     if ($value =~ m/[\n\r]/) {
2434         die "property contains a line feed\n";
2435     }
2436
2437     if ($type eq 'boolean') {
2438         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2439         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2440         die "type check ('boolean') failed - got '$value'\n";
2441     } elsif ($type eq 'integer') {
2442         return int($1) if $value =~ m/^(\d+)$/;
2443         die "type check ('integer') failed - got '$value'\n";
2444     } elsif ($type eq 'number') {
2445         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2446         die "type check ('number') failed - got '$value'\n";
2447     } elsif ($type eq 'string') {
2448         if (my $fmt = $confdesc->{$key}->{format}) {
2449             PVE::JSONSchema::check_format($fmt, $value);
2450             return $value;
2451         }
2452         $value =~ s/^\"(.*)\"$/$1/;
2453         return $value;
2454     } else {
2455         die "internal error"
2456     }
2457 }
2458
2459 sub touch_config {
2460     my ($vmid) = @_;
2461
2462     my $conf = PVE::QemuConfig->config_file($vmid);
2463     utime undef, undef, $conf;
2464 }
2465
2466 sub destroy_vm {
2467     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2468
2469     my $conffile = PVE::QemuConfig->config_file($vmid);
2470
2471     my $conf = PVE::QemuConfig->load_config($vmid);
2472
2473     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2474
2475     if ($conf->{template}) {
2476         # check if any base image is still used by a linked clone
2477         foreach_drive($conf, sub {
2478                 my ($ds, $drive) = @_;
2479
2480                 return if drive_is_cdrom($drive);
2481
2482                 my $volid = $drive->{file};
2483
2484                 return if !$volid || $volid =~ m|^/|;
2485
2486                 die "base volume '$volid' is still in use by linked cloned\n"
2487                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2488
2489         });
2490     }
2491
2492     # only remove disks owned by this VM
2493     foreach_drive($conf, sub {
2494         my ($ds, $drive) = @_;
2495
2496         return if drive_is_cdrom($drive, 1);
2497
2498         my $volid = $drive->{file};
2499
2500         return if !$volid || $volid =~ m|^/|;
2501
2502         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2503         return if !$path || !$owner || ($owner != $vmid);
2504
2505         eval {
2506             PVE::Storage::vdisk_free($storecfg, $volid);
2507         };
2508         warn "Could not remove disk '$volid', check manually: $@" if $@;
2509
2510     });
2511
2512     if ($keep_empty_config) {
2513         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2514     } else {
2515         unlink $conffile;
2516     }
2517
2518     # also remove unused disk
2519     eval {
2520         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2521
2522         eval {
2523             PVE::Storage::foreach_volid($dl, sub {
2524                 my ($volid, $sid, $volname, $d) = @_;
2525                 PVE::Storage::vdisk_free($storecfg, $volid);
2526             });
2527         };
2528         warn $@ if $@;
2529
2530     };
2531     warn $@ if $@;
2532 }
2533
2534 sub parse_vm_config {
2535     my ($filename, $raw) = @_;
2536
2537     return undef if !defined($raw);
2538
2539     my $res = {
2540         digest => Digest::SHA::sha1_hex($raw),
2541         snapshots => {},
2542         pending => {},
2543     };
2544
2545     $filename =~ m|/qemu-server/(\d+)\.conf$|
2546         || die "got strange filename '$filename'";
2547
2548     my $vmid = $1;
2549
2550     my $conf = $res;
2551     my $descr;
2552     my $section = '';
2553
2554     my @lines = split(/\n/, $raw);
2555     foreach my $line (@lines) {
2556         next if $line =~ m/^\s*$/;
2557
2558         if ($line =~ m/^\[PENDING\]\s*$/i) {
2559             $section = 'pending';
2560             if (defined($descr)) {
2561                 $descr =~ s/\s+$//;
2562                 $conf->{description} = $descr;
2563             }
2564             $descr = undef;
2565             $conf = $res->{$section} = {};
2566             next;
2567
2568         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2569             $section = $1;
2570             if (defined($descr)) {
2571                 $descr =~ s/\s+$//;
2572                 $conf->{description} = $descr;
2573             }
2574             $descr = undef;
2575             $conf = $res->{snapshots}->{$section} = {};
2576             next;
2577         }
2578
2579         if ($line =~ m/^\#(.*)\s*$/) {
2580             $descr = '' if !defined($descr);
2581             $descr .= PVE::Tools::decode_text($1) . "\n";
2582             next;
2583         }
2584
2585         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2586             $descr = '' if !defined($descr);
2587             $descr .= PVE::Tools::decode_text($2);
2588         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2589             $conf->{snapstate} = $1;
2590         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2591             my $key = $1;
2592             my $value = $2;
2593             $conf->{$key} = $value;
2594         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2595             my $value = $1;
2596             if ($section eq 'pending') {
2597                 $conf->{delete} = $value; # we parse this later
2598             } else {
2599                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2600             }
2601         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2602             my $key = $1;
2603             my $value = $2;
2604             eval { $value = check_type($key, $value); };
2605             if ($@) {
2606                 warn "vm $vmid - unable to parse value of '$key' - $@";
2607             } else {
2608                 $key = 'ide2' if $key eq 'cdrom';
2609                 my $fmt = $confdesc->{$key}->{format};
2610                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2611                     my $v = parse_drive($key, $value);
2612                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2613                         $v->{file} = $volid;
2614                         $value = print_drive($vmid, $v);
2615                     } else {
2616                         warn "vm $vmid - unable to parse value of '$key'\n";
2617                         next;
2618                     }
2619                 }
2620
2621                 $conf->{$key} = $value;
2622             }
2623         }
2624     }
2625
2626     if (defined($descr)) {
2627         $descr =~ s/\s+$//;
2628         $conf->{description} = $descr;
2629     }
2630     delete $res->{snapstate}; # just to be sure
2631
2632     return $res;
2633 }
2634
2635 sub write_vm_config {
2636     my ($filename, $conf) = @_;
2637
2638     delete $conf->{snapstate}; # just to be sure
2639
2640     if ($conf->{cdrom}) {
2641         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2642         $conf->{ide2} = $conf->{cdrom};
2643         delete $conf->{cdrom};
2644     }
2645
2646     # we do not use 'smp' any longer
2647     if ($conf->{sockets}) {
2648         delete $conf->{smp};
2649     } elsif ($conf->{smp}) {
2650         $conf->{sockets} = $conf->{smp};
2651         delete $conf->{cores};
2652         delete $conf->{smp};
2653     }
2654
2655     my $used_volids = {};
2656
2657     my $cleanup_config = sub {
2658         my ($cref, $pending, $snapname) = @_;
2659
2660         foreach my $key (keys %$cref) {
2661             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2662                 $key eq 'snapstate' || $key eq 'pending';
2663             my $value = $cref->{$key};
2664             if ($key eq 'delete') {
2665                 die "propertry 'delete' is only allowed in [PENDING]\n"
2666                     if !$pending;
2667                 # fixme: check syntax?
2668                 next;
2669             }
2670             eval { $value = check_type($key, $value); };
2671             die "unable to parse value of '$key' - $@" if $@;
2672
2673             $cref->{$key} = $value;
2674
2675             if (!$snapname && is_valid_drivename($key)) {
2676                 my $drive = parse_drive($key, $value);
2677                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2678             }
2679         }
2680     };
2681
2682     &$cleanup_config($conf);
2683
2684     &$cleanup_config($conf->{pending}, 1);
2685
2686     foreach my $snapname (keys %{$conf->{snapshots}}) {
2687         die "internal error" if $snapname eq 'pending';
2688         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2689     }
2690
2691     # remove 'unusedX' settings if we re-add a volume
2692     foreach my $key (keys %$conf) {
2693         my $value = $conf->{$key};
2694         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2695             delete $conf->{$key};
2696         }
2697     }
2698
2699     my $generate_raw_config = sub {
2700         my ($conf, $pending) = @_;
2701
2702         my $raw = '';
2703
2704         # add description as comment to top of file
2705         if (defined(my $descr = $conf->{description})) {
2706             if ($descr) {
2707                 foreach my $cl (split(/\n/, $descr)) {
2708                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2709                 }
2710             } else {
2711                 $raw .= "#\n" if $pending;
2712             }
2713         }
2714
2715         foreach my $key (sort keys %$conf) {
2716             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2717             $raw .= "$key: $conf->{$key}\n";
2718         }
2719         return $raw;
2720     };
2721
2722     my $raw = &$generate_raw_config($conf);
2723
2724     if (scalar(keys %{$conf->{pending}})){
2725         $raw .= "\n[PENDING]\n";
2726         $raw .= &$generate_raw_config($conf->{pending}, 1);
2727     }
2728
2729     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2730         $raw .= "\n[$snapname]\n";
2731         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2732     }
2733
2734     return $raw;
2735 }
2736
2737 sub load_defaults {
2738
2739     my $res = {};
2740
2741     # we use static defaults from our JSON schema configuration
2742     foreach my $key (keys %$confdesc) {
2743         if (defined(my $default = $confdesc->{$key}->{default})) {
2744             $res->{$key} = $default;
2745         }
2746     }
2747
2748     return $res;
2749 }
2750
2751 sub config_list {
2752     my $vmlist = PVE::Cluster::get_vmlist();
2753     my $res = {};
2754     return $res if !$vmlist || !$vmlist->{ids};
2755     my $ids = $vmlist->{ids};
2756
2757     foreach my $vmid (keys %$ids) {
2758         my $d = $ids->{$vmid};
2759         next if !$d->{node} || $d->{node} ne $nodename;
2760         next if !$d->{type} || $d->{type} ne 'qemu';
2761         $res->{$vmid}->{exists} = 1;
2762     }
2763     return $res;
2764 }
2765
2766 # test if VM uses local resources (to prevent migration)
2767 sub check_local_resources {
2768     my ($conf, $noerr) = @_;
2769
2770     my $loc_res = 0;
2771
2772     $loc_res = 1 if $conf->{hostusb}; # old syntax
2773     $loc_res = 1 if $conf->{hostpci}; # old syntax
2774
2775     foreach my $k (keys %$conf) {
2776         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2777         # sockets are safe: they will recreated be on the target side post-migrate
2778         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2779         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2780     }
2781
2782     die "VM uses local resources\n" if $loc_res && !$noerr;
2783
2784     return $loc_res;
2785 }
2786
2787 # check if used storages are available on all nodes (use by migrate)
2788 sub check_storage_availability {
2789     my ($storecfg, $conf, $node) = @_;
2790
2791     foreach_drive($conf, sub {
2792         my ($ds, $drive) = @_;
2793
2794         my $volid = $drive->{file};
2795         return if !$volid;
2796
2797         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2798         return if !$sid;
2799
2800         # check if storage is available on both nodes
2801         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2802         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2803    });
2804 }
2805
2806 # list nodes where all VM images are available (used by has_feature API)
2807 sub shared_nodes {
2808     my ($conf, $storecfg) = @_;
2809
2810     my $nodelist = PVE::Cluster::get_nodelist();
2811     my $nodehash = { map { $_ => 1 } @$nodelist };
2812     my $nodename = PVE::INotify::nodename();
2813
2814     foreach_drive($conf, sub {
2815         my ($ds, $drive) = @_;
2816
2817         my $volid = $drive->{file};
2818         return if !$volid;
2819
2820         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2821         if ($storeid) {
2822             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2823             if ($scfg->{disable}) {
2824                 $nodehash = {};
2825             } elsif (my $avail = $scfg->{nodes}) {
2826                 foreach my $node (keys %$nodehash) {
2827                     delete $nodehash->{$node} if !$avail->{$node};
2828                 }
2829             } elsif (!$scfg->{shared}) {
2830                 foreach my $node (keys %$nodehash) {
2831                     delete $nodehash->{$node} if $node ne $nodename
2832                 }
2833             }
2834         }
2835     });
2836
2837     return $nodehash
2838 }
2839
2840 sub check_cmdline {
2841     my ($pidfile, $pid) = @_;
2842
2843     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2844     if (defined($fh)) {
2845         my $line = <$fh>;
2846         $fh->close;
2847         return undef if !$line;
2848         my @param = split(/\0/, $line);
2849
2850         my $cmd = $param[0];
2851         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2852
2853         for (my $i = 0; $i < scalar (@param); $i++) {
2854             my $p = $param[$i];
2855             next if !$p;
2856             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2857                 my $p = $param[$i+1];
2858                 return 1 if $p && ($p eq $pidfile);
2859                 return undef;
2860             }
2861         }
2862     }
2863     return undef;
2864 }
2865
2866 sub check_running {
2867     my ($vmid, $nocheck, $node) = @_;
2868
2869     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2870
2871     die "unable to find configuration file for VM $vmid - no such machine\n"
2872         if !$nocheck && ! -f $filename;
2873
2874     my $pidfile = pidfile_name($vmid);
2875
2876     if (my $fd = IO::File->new("<$pidfile")) {
2877         my $st = stat($fd);
2878         my $line = <$fd>;
2879         close($fd);
2880
2881         my $mtime = $st->mtime;
2882         if ($mtime > time()) {
2883             warn "file '$filename' modified in future\n";
2884         }
2885
2886         if ($line =~ m/^(\d+)$/) {
2887             my $pid = $1;
2888             if (check_cmdline($pidfile, $pid)) {
2889                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2890                     return $pid;
2891                 }
2892             }
2893         }
2894     }
2895
2896     return undef;
2897 }
2898
2899 sub vzlist {
2900
2901     my $vzlist = config_list();
2902
2903     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2904
2905     while (defined(my $de = $fd->read)) {
2906         next if $de !~ m/^(\d+)\.pid$/;
2907         my $vmid = $1;
2908         next if !defined($vzlist->{$vmid});
2909         if (my $pid = check_running($vmid)) {
2910             $vzlist->{$vmid}->{pid} = $pid;
2911         }
2912     }
2913
2914     return $vzlist;
2915 }
2916
2917 sub disksize {
2918     my ($storecfg, $conf) = @_;
2919
2920     my $bootdisk = $conf->{bootdisk};
2921     return undef if !$bootdisk;
2922     return undef if !is_valid_drivename($bootdisk);
2923
2924     return undef if !$conf->{$bootdisk};
2925
2926     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2927     return undef if !defined($drive);
2928
2929     return undef if drive_is_cdrom($drive);
2930
2931     my $volid = $drive->{file};
2932     return undef if !$volid;
2933
2934     return $drive->{size};
2935 }
2936
2937 our $vmstatus_return_properties = {
2938     vmid => get_standard_option('pve-vmid'),
2939     status => {
2940         description => "Qemu process status.",
2941         type => 'string',
2942         enum => ['stopped', 'running'],
2943     },
2944     maxmem => {
2945         description => "Maximum memory in bytes.",
2946         type => 'integer',
2947         optional => 1,
2948         renderer => 'bytes',
2949     },
2950     maxdisk => {
2951         description => "Root disk size in bytes.",
2952         type => 'integer',
2953         optional => 1,
2954         renderer => 'bytes',
2955     },
2956     name => {
2957         description => "VM name.",
2958         type => 'string',
2959         optional => 1,
2960     },
2961     qmpstatus => {
2962         description => "Qemu QMP agent status.",
2963         type => 'string',
2964         optional => 1,
2965     },
2966     pid => {
2967         description => "PID of running qemu process.",
2968         type => 'integer',
2969         optional => 1,
2970     },
2971     uptime => {
2972         description => "Uptime.",
2973         type => 'integer',
2974         optional => 1,
2975         renderer => 'duration',
2976     },
2977     cpus => {
2978         description => "Maximum usable CPUs.",
2979         type => 'number',
2980         optional => 1,
2981     },
2982 };
2983
2984 my $last_proc_pid_stat;
2985
2986 # get VM status information
2987 # This must be fast and should not block ($full == false)
2988 # We only query KVM using QMP if $full == true (this can be slow)
2989 sub vmstatus {
2990     my ($opt_vmid, $full) = @_;
2991
2992     my $res = {};
2993
2994     my $storecfg = PVE::Storage::config();
2995
2996     my $list = vzlist();
2997     my $defaults = load_defaults();
2998
2999     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3000
3001     my $cpucount = $cpuinfo->{cpus} || 1;
3002
3003     foreach my $vmid (keys %$list) {
3004         next if $opt_vmid && ($vmid ne $opt_vmid);
3005
3006         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3007         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3008
3009         my $d = { vmid => $vmid };
3010         $d->{pid} = $list->{$vmid}->{pid};
3011
3012         # fixme: better status?
3013         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3014
3015         my $size = disksize($storecfg, $conf);
3016         if (defined($size)) {
3017             $d->{disk} = 0; # no info available
3018             $d->{maxdisk} = $size;
3019         } else {
3020             $d->{disk} = 0;
3021             $d->{maxdisk} = 0;
3022         }
3023
3024         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3025             * ($conf->{cores} || $defaults->{cores});
3026         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3027         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3028
3029         $d->{name} = $conf->{name} || "VM $vmid";
3030         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3031             : $defaults->{memory}*(1024*1024);
3032
3033         if ($conf->{balloon}) {
3034             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3035             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3036                 : $defaults->{shares};
3037         }
3038
3039         $d->{uptime} = 0;
3040         $d->{cpu} = 0;
3041         $d->{mem} = 0;
3042
3043         $d->{netout} = 0;
3044         $d->{netin} = 0;
3045
3046         $d->{diskread} = 0;
3047         $d->{diskwrite} = 0;
3048
3049         $d->{template} = PVE::QemuConfig->is_template($conf);
3050
3051         $d->{serial} = 1 if conf_has_serial($conf);
3052
3053         $res->{$vmid} = $d;
3054     }
3055
3056     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3057     foreach my $dev (keys %$netdev) {
3058         next if $dev !~ m/^tap([1-9]\d*)i/;
3059         my $vmid = $1;
3060         my $d = $res->{$vmid};
3061         next if !$d;
3062
3063         $d->{netout} += $netdev->{$dev}->{receive};
3064         $d->{netin} += $netdev->{$dev}->{transmit};
3065
3066         if ($full) {
3067             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3068             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3069         }
3070
3071     }
3072
3073     my $ctime = gettimeofday;
3074
3075     foreach my $vmid (keys %$list) {
3076
3077         my $d = $res->{$vmid};
3078         my $pid = $d->{pid};
3079         next if !$pid;
3080
3081         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3082         next if !$pstat; # not running
3083
3084         my $used = $pstat->{utime} + $pstat->{stime};
3085
3086         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3087
3088         if ($pstat->{vsize}) {
3089             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3090         }
3091
3092         my $old = $last_proc_pid_stat->{$pid};
3093         if (!$old) {
3094             $last_proc_pid_stat->{$pid} = {
3095                 time => $ctime,
3096                 used => $used,
3097                 cpu => 0,
3098             };
3099             next;
3100         }
3101
3102         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3103
3104         if ($dtime > 1000) {
3105             my $dutime = $used -  $old->{used};
3106
3107             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3108             $last_proc_pid_stat->{$pid} = {
3109                 time => $ctime,
3110                 used => $used,
3111                 cpu => $d->{cpu},
3112             };
3113         } else {
3114             $d->{cpu} = $old->{cpu};
3115         }
3116     }
3117
3118     return $res if !$full;
3119
3120     my $qmpclient = PVE::QMPClient->new();
3121
3122     my $ballooncb = sub {
3123         my ($vmid, $resp) = @_;
3124
3125         my $info = $resp->{'return'};
3126         return if !$info->{max_mem};
3127
3128         my $d = $res->{$vmid};
3129
3130         # use memory assigned to VM
3131         $d->{maxmem} = $info->{max_mem};
3132         $d->{balloon} = $info->{actual};
3133
3134         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3135             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3136             $d->{freemem} = $info->{free_mem};
3137         }
3138
3139         $d->{ballooninfo} = $info;
3140     };
3141
3142     my $blockstatscb = sub {
3143         my ($vmid, $resp) = @_;
3144         my $data = $resp->{'return'} || [];
3145         my $totalrdbytes = 0;
3146         my $totalwrbytes = 0;
3147
3148         for my $blockstat (@$data) {
3149             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3150             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3151
3152             $blockstat->{device} =~ s/drive-//;
3153             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3154         }
3155         $res->{$vmid}->{diskread} = $totalrdbytes;
3156         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3157     };
3158
3159     my $statuscb = sub {
3160         my ($vmid, $resp) = @_;
3161
3162         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3163         # this fails if ballon driver is not loaded, so this must be
3164         # the last commnand (following command are aborted if this fails).
3165         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3166
3167         my $status = 'unknown';
3168         if (!defined($status = $resp->{'return'}->{status})) {
3169             warn "unable to get VM status\n";
3170             return;
3171         }
3172
3173         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3174     };
3175
3176     foreach my $vmid (keys %$list) {
3177         next if $opt_vmid && ($vmid ne $opt_vmid);
3178         next if !$res->{$vmid}->{pid}; # not running
3179         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3180     }
3181
3182     $qmpclient->queue_execute(undef, 2);
3183
3184     foreach my $vmid (keys %$list) {
3185         next if $opt_vmid && ($vmid ne $opt_vmid);
3186         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3187     }
3188
3189     return $res;
3190 }
3191
3192 sub foreach_drive {
3193     my ($conf, $func, @param) = @_;
3194
3195     foreach my $ds (valid_drive_names()) {
3196         next if !defined($conf->{$ds});
3197
3198         my $drive = parse_drive($ds, $conf->{$ds});
3199         next if !$drive;
3200
3201         &$func($ds, $drive, @param);
3202     }
3203 }
3204
3205 sub foreach_volid {
3206     my ($conf, $func, @param) = @_;
3207
3208     my $volhash = {};
3209
3210     my $test_volid = sub {
3211         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3212
3213         return if !$volid;
3214
3215         $volhash->{$volid}->{cdrom} //= 1;
3216         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3217
3218         $volhash->{$volid}->{replicate} //= 0;
3219         $volhash->{$volid}->{replicate} = 1 if $replicate;
3220
3221         $volhash->{$volid}->{shared} //= 0;
3222         $volhash->{$volid}->{shared} = 1 if $shared;
3223
3224         $volhash->{$volid}->{referenced_in_config} //= 0;
3225         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3226
3227         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3228             if defined($snapname);
3229     };
3230
3231     foreach_drive($conf, sub {
3232         my ($ds, $drive) = @_;
3233         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3234     });
3235
3236     foreach my $snapname (keys %{$conf->{snapshots}}) {
3237         my $snap = $conf->{snapshots}->{$snapname};
3238         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3239         foreach_drive($snap, sub {
3240             my ($ds, $drive) = @_;
3241             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3242         });
3243     }
3244
3245     foreach my $volid (keys %$volhash) {
3246         &$func($volid, $volhash->{$volid}, @param);
3247     }
3248 }
3249
3250 sub conf_has_serial {
3251     my ($conf) = @_;
3252
3253     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3254         if ($conf->{"serial$i"}) {
3255             return 1;
3256         }
3257     }
3258
3259     return 0;
3260 }
3261
3262 sub vga_conf_has_spice {
3263     my ($vga) = @_;
3264
3265     my $vgaconf = parse_vga($vga);
3266     my $vgatype = $vgaconf->{type};
3267     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3268
3269     return $1 || 1;
3270 }
3271
3272 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3273 sub get_host_arch() {
3274     $host_arch = (POSIX::uname())[4] if !$host_arch;
3275     return $host_arch;
3276 }
3277
3278 sub is_native($) {
3279     my ($arch) = @_;
3280     return get_host_arch() eq $arch;
3281 }
3282
3283 my $default_machines = {
3284     x86_64 => 'pc',
3285     aarch64 => 'virt',
3286 };
3287
3288 sub get_basic_machine_info {
3289     my ($conf, $forcemachine) = @_;
3290
3291     my $arch = $conf->{arch} // get_host_arch();
3292     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3293     return ($arch, $machine);
3294 }
3295
3296 sub get_ovmf_files($) {
3297     my ($arch) = @_;
3298
3299     my $ovmf = $OVMF->{$arch}
3300         or die "no OVMF images known for architecture '$arch'\n";
3301
3302     return @$ovmf;
3303 }
3304
3305 my $Arch2Qemu = {
3306     aarch64 => '/usr/bin/qemu-system-aarch64',
3307     x86_64 => '/usr/bin/qemu-system-x86_64',
3308 };
3309 sub get_command_for_arch($) {
3310     my ($arch) = @_;
3311     return '/usr/bin/kvm' if is_native($arch);
3312
3313     my $cmd = $Arch2Qemu->{$arch}
3314         or die "don't know how to emulate architecture '$arch'\n";
3315     return $cmd;
3316 }
3317
3318 sub get_cpu_options {
3319     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3320
3321     my $cpuFlags = [];
3322     my $ostype = $conf->{ostype};
3323
3324     my $cpu = $kvm ? "kvm64" : "qemu64";
3325     if ($arch eq 'aarch64') {
3326         $cpu = 'cortex-a57';
3327     }
3328     if (my $cputype = $conf->{cpu}) {
3329         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3330             or die "Cannot parse cpu description: $cputype\n";
3331         $cpu = $cpuconf->{cputype};
3332         $kvm_off = 1 if $cpuconf->{hidden};
3333
3334         if (defined(my $flags = $cpuconf->{flags})) {
3335             push @$cpuFlags, split(";", $flags);
3336         }
3337     }
3338
3339     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3340
3341     push @$cpuFlags , '-x2apic'
3342         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3343
3344     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3345
3346     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3347
3348     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3349
3350         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3351         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3352     }
3353
3354     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3355
3356     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3357
3358     push @$cpuFlags, 'kvm=off' if $kvm_off;
3359
3360     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3361         push @$cpuFlags, "vendor=${cpu_vendor}"
3362             if $cpu_vendor ne 'default';
3363     } elsif ($arch ne 'aarch64') {
3364         die "internal error"; # should not happen
3365     }
3366
3367     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3368
3369     return ('-cpu', $cpu);
3370 }
3371
3372 sub config_to_command {
3373     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3374
3375     my $cmd = [];
3376     my $globalFlags = [];
3377     my $machineFlags = [];
3378     my $rtcFlags = [];
3379     my $devices = [];
3380     my $pciaddr = '';
3381     my $bridges = {};
3382     my $kvmver = kvm_user_version();
3383     my $vernum = 0; # unknown
3384     my $ostype = $conf->{ostype};
3385     my $winversion = windows_version($ostype);
3386     my $kvm = $conf->{kvm};
3387
3388     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3389     $kvm //= 1 if is_native($arch);
3390
3391     if ($kvm) {
3392         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3393             if !defined kvm_version();
3394     }
3395
3396     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3397         $vernum = $1*1000000+$2*1000;
3398     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3399         $vernum = $1*1000000+$2*1000+$3;
3400     }
3401
3402     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3403
3404     my $have_ovz = -f '/proc/vz/vestat';
3405
3406     my $q35 = machine_type_is_q35($conf);
3407     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3408     my $use_old_bios_files = undef;
3409     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3410
3411     my $cpuunits = defined($conf->{cpuunits}) ?
3412             $conf->{cpuunits} : $defaults->{cpuunits};
3413
3414     push @$cmd, get_command_for_arch($arch);
3415
3416     push @$cmd, '-id', $vmid;
3417
3418     my $vmname = $conf->{name} || "vm$vmid";
3419
3420     push @$cmd, '-name', $vmname;
3421
3422     my $use_virtio = 0;
3423
3424     my $qmpsocket = qmp_socket($vmid);
3425     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3426     push @$cmd, '-mon', "chardev=qmp,mode=control";
3427
3428     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3429         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3430         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3431     }
3432
3433     push @$cmd, '-pidfile' , pidfile_name($vmid);
3434
3435     push @$cmd, '-daemonize';
3436
3437     if ($conf->{smbios1}) {
3438         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3439     }
3440
3441     if ($conf->{vmgenid}) {
3442         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3443     }
3444
3445     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3446     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3447         die "uefi base image not found\n" if ! -f $ovmf_code;
3448
3449         my $path;
3450         my $format;
3451         if (my $efidisk = $conf->{efidisk0}) {
3452             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3453             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3454             $format = $d->{format};
3455             if ($storeid) {
3456                 $path = PVE::Storage::path($storecfg, $d->{file});
3457                 if (!defined($format)) {
3458                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3459                     $format = qemu_img_format($scfg, $volname);
3460                 }
3461             } else {
3462                 $path = $d->{file};
3463                 die "efidisk format must be specified\n"
3464                     if !defined($format);
3465             }
3466         } else {
3467             warn "no efidisk configured! Using temporary efivars disk.\n";
3468             $path = "/tmp/$vmid-ovmf.fd";
3469             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3470             $format = 'raw';
3471         }
3472
3473         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3474         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3475     }
3476
3477
3478     # add usb controllers
3479     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3480     push @$devices, @usbcontrollers if @usbcontrollers;
3481     my $vga = parse_vga($conf->{vga});
3482
3483     my $qxlnum = vga_conf_has_spice($conf->{vga});
3484     $vga->{type} = 'qxl' if $qxlnum;
3485
3486     if (!$vga->{type}) {
3487         if ($arch eq 'aarch64') {
3488             $vga->{type} = 'virtio';
3489         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3490             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3491         } else {
3492             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3493         }
3494     }
3495
3496     # enable absolute mouse coordinates (needed by vnc)
3497     my $tablet;
3498     if (defined($conf->{tablet})) {
3499         $tablet = $conf->{tablet};
3500     } else {
3501         $tablet = $defaults->{tablet};
3502         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3503         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3504     }
3505
3506     if ($tablet) {
3507         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3508         my $kbd = print_keyboarddevice_full($conf, $arch);
3509         push @$devices, '-device', $kbd if defined($kbd);
3510     }
3511
3512     my $kvm_off = 0;
3513     my $gpu_passthrough;
3514
3515     # host pci devices
3516     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3517         my $d = parse_hostpci($conf->{"hostpci$i"});
3518         next if !$d;
3519
3520         my $pcie = $d->{pcie};
3521         if($pcie){
3522             die "q35 machine model is not enabled" if !$q35;
3523             $pciaddr = print_pcie_addr("hostpci$i");
3524         }else{
3525             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3526         }
3527
3528         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3529         my $romfile = $d->{romfile};
3530
3531         my $xvga = '';
3532         if ($d->{'x-vga'}) {
3533             $xvga = ',x-vga=on';
3534             $kvm_off = 1;
3535             $vga->{type} = 'none';
3536             $gpu_passthrough = 1;
3537
3538             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3539                 $xvga = "";
3540             }
3541         }
3542         my $pcidevices = $d->{pciid};
3543         my $multifunction = 1 if @$pcidevices > 1;
3544
3545         my $j=0;
3546         foreach my $pcidevice (@$pcidevices) {
3547
3548             my $id = "hostpci$i";
3549             $id .= ".$j" if $multifunction;
3550             my $addr = $pciaddr;
3551             $addr .= ".$j" if $multifunction;
3552             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3553
3554             if($j == 0){
3555                 $devicestr .= "$rombar$xvga";
3556                 $devicestr .= ",multifunction=on" if $multifunction;
3557                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3558             }
3559
3560             push @$devices, '-device', $devicestr;
3561             $j++;
3562         }
3563     }
3564
3565     # usb devices
3566     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3567     push @$devices, @usbdevices if @usbdevices;
3568     # serial devices
3569     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3570         if (my $path = $conf->{"serial$i"}) {
3571             if ($path eq 'socket') {
3572                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3573                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3574                 # On aarch64, serial0 is the UART device. Qemu only allows
3575                 # connecting UART devices via the '-serial' command line, as
3576                 # the device has a fixed slot on the hardware...
3577                 if ($arch eq 'aarch64' && $i == 0) {
3578                     push @$devices, '-serial', "chardev:serial$i";
3579                 } else {
3580                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3581                 }
3582             } else {
3583                 die "no such serial device\n" if ! -c $path;
3584                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3585                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3586             }
3587         }
3588     }
3589
3590     # parallel devices
3591     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3592         if (my $path = $conf->{"parallel$i"}) {
3593             die "no such parallel device\n" if ! -c $path;
3594             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3595             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3596             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3597         }
3598     }
3599
3600
3601     my $sockets = 1;
3602     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3603     $sockets = $conf->{sockets} if  $conf->{sockets};
3604
3605     my $cores = $conf->{cores} || 1;
3606
3607     my $maxcpus = $sockets * $cores;
3608
3609     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3610
3611     my $allowed_vcpus = $cpuinfo->{cpus};
3612
3613     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3614         if ($allowed_vcpus < $maxcpus);
3615
3616     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3617
3618         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3619         for (my $i = 2; $i <= $vcpus; $i++)  {
3620             my $cpustr = print_cpu_device($conf,$i);
3621             push @$cmd, '-device', $cpustr;
3622         }
3623
3624     } else {
3625
3626         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3627     }
3628     push @$cmd, '-nodefaults';
3629
3630     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3631
3632     my $bootindex_hash = {};
3633     my $i = 1;
3634     foreach my $o (split(//, $bootorder)) {
3635         $bootindex_hash->{$o} = $i*100;
3636         $i++;
3637     }
3638
3639     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3640
3641     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3642
3643     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3644
3645     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3646         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3647         my $socket = vnc_socket($vmid);
3648         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3649     } else {
3650         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3651         push @$cmd, '-nographic';
3652     }
3653
3654     # time drift fix
3655     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3656
3657     my $useLocaltime = $conf->{localtime};
3658
3659     if ($winversion >= 5) { # windows
3660         $useLocaltime = 1 if !defined($conf->{localtime});
3661
3662         # use time drift fix when acpi is enabled
3663         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3664             $tdf = 1 if !defined($conf->{tdf});
3665         }
3666     }
3667
3668     if ($winversion >= 6) {
3669         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3670         push @$cmd, '-no-hpet';
3671     }
3672
3673     push @$rtcFlags, 'driftfix=slew' if $tdf;
3674
3675     if (!$kvm) {
3676         push @$machineFlags, 'accel=tcg';
3677     }
3678
3679     if ($machine_type) {
3680         push @$machineFlags, "type=${machine_type}";
3681     }
3682
3683     if ($conf->{startdate}) {
3684         push @$rtcFlags, "base=$conf->{startdate}";
3685     } elsif ($useLocaltime) {
3686         push @$rtcFlags, 'base=localtime';
3687     }
3688
3689     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3690
3691     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3692
3693     push @$cmd, '-S' if $conf->{freeze};
3694
3695     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3696
3697     # enable sound
3698     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3699     #push @$cmd, '-soundhw', 'es1370';
3700     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3701
3702     if (parse_guest_agent($conf)->{enabled}) {
3703         my $qgasocket = qmp_socket($vmid, 1);
3704         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3705         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3706         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3707         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3708     }
3709
3710     my $spice_port;
3711
3712     if ($qxlnum) {
3713         if ($qxlnum > 1) {
3714             if ($winversion){
3715                 for(my $i = 1; $i < $qxlnum; $i++){
3716                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3717                 }
3718             } else {
3719                 # assume other OS works like Linux
3720                 my ($ram, $vram) = ("134217728", "67108864");
3721                 if ($vga->{memory}) {
3722                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3723                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3724                 }
3725                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3726                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3727             }
3728         }
3729
3730         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3731
3732         my $nodename = PVE::INotify::nodename();
3733         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3734         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3735         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3736         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3737         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3738
3739         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3740
3741         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3742         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3743         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3744     }
3745
3746     # enable balloon by default, unless explicitly disabled
3747     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3748         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3749         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3750     }
3751
3752     if ($conf->{watchdog}) {
3753         my $wdopts = parse_watchdog($conf->{watchdog});
3754         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3755         my $watchdog = $wdopts->{model} || 'i6300esb';
3756         push @$devices, '-device', "$watchdog$pciaddr";
3757         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3758     }
3759
3760     my $vollist = [];
3761     my $scsicontroller = {};
3762     my $ahcicontroller = {};
3763     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3764
3765     # Add iscsi initiator name if available
3766     if (my $initiator = get_initiator_name()) {
3767         push @$devices, '-iscsi', "initiator-name=$initiator";
3768     }
3769
3770     foreach_drive($conf, sub {
3771         my ($ds, $drive) = @_;
3772
3773         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3774             push @$vollist, $drive->{file};
3775         }
3776
3777         # ignore efidisk here, already added in bios/fw handling code above
3778         return if $drive->{interface} eq 'efidisk';
3779
3780         $use_virtio = 1 if $ds =~ m/^virtio/;
3781
3782         if (drive_is_cdrom ($drive)) {
3783             if ($bootindex_hash->{d}) {
3784                 $drive->{bootindex} = $bootindex_hash->{d};
3785                 $bootindex_hash->{d} += 1;
3786             }
3787         } else {
3788             if ($bootindex_hash->{c}) {
3789                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3790                 $bootindex_hash->{c} += 1;
3791             }
3792         }
3793
3794         if($drive->{interface} eq 'virtio'){
3795            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3796         }
3797
3798         if ($drive->{interface} eq 'scsi') {
3799
3800             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3801
3802             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3803             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3804
3805             my $iothread = '';
3806             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3807                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3808                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3809             } elsif ($drive->{iothread}) {
3810                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3811             }
3812
3813             my $queues = '';
3814             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3815                 $queues = ",num_queues=$drive->{queues}";
3816             }
3817
3818             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3819             $scsicontroller->{$controller}=1;
3820         }
3821
3822         if ($drive->{interface} eq 'sata') {
3823            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3824            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3825            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3826            $ahcicontroller->{$controller}=1;
3827         }
3828
3829         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3830         push @$devices, '-drive',$drive_cmd;
3831         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3832     });
3833
3834     for (my $i = 0; $i < $MAX_NETS; $i++) {
3835          next if !$conf->{"net$i"};
3836          my $d = parse_net($conf->{"net$i"});
3837          next if !$d;
3838
3839          $use_virtio = 1 if $d->{model} eq 'virtio';
3840
3841          if ($bootindex_hash->{n}) {
3842             $d->{bootindex} = $bootindex_hash->{n};
3843             $bootindex_hash->{n} += 1;
3844          }
3845
3846          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3847          push @$devices, '-netdev', $netdevfull;
3848
3849          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3850          push @$devices, '-device', $netdevicefull;
3851     }
3852
3853     if (!$q35) {
3854         # add pci bridges
3855         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3856            $bridges->{1} = 1;
3857            $bridges->{2} = 1;
3858         }
3859
3860         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3861
3862         while (my ($k, $v) = each %$bridges) {
3863             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3864             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3865         }
3866     }
3867
3868     # add custom args
3869     if ($conf->{args}) {
3870         my $aa = PVE::Tools::split_args($conf->{args});
3871         push @$cmd, @$aa;
3872     }
3873
3874     push @$cmd, @$devices;
3875     push @$cmd, '-rtc', join(',', @$rtcFlags)
3876         if scalar(@$rtcFlags);
3877     push @$cmd, '-machine', join(',', @$machineFlags)
3878         if scalar(@$machineFlags);
3879     push @$cmd, '-global', join(',', @$globalFlags)
3880         if scalar(@$globalFlags);
3881
3882     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3883 }
3884
3885 sub vnc_socket {
3886     my ($vmid) = @_;
3887     return "${var_run_tmpdir}/$vmid.vnc";
3888 }
3889
3890 sub spice_port {
3891     my ($vmid) = @_;
3892
3893     my $res = vm_mon_cmd($vmid, 'query-spice');
3894
3895     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3896 }
3897
3898 sub qmp_socket {
3899     my ($vmid, $qga, $name) = @_;
3900     my $sockettype = $qga ? 'qga' : 'qmp';
3901     my $ext = $name ? '-'.$name : '';
3902     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3903 }
3904
3905 sub pidfile_name {
3906     my ($vmid) = @_;
3907     return "${var_run_tmpdir}/$vmid.pid";
3908 }
3909
3910 sub vm_devices_list {
3911     my ($vmid) = @_;
3912
3913     my $res = vm_mon_cmd($vmid, 'query-pci');
3914     my $devices_to_check = [];
3915     my $devices = {};
3916     foreach my $pcibus (@$res) {
3917         push @$devices_to_check, @{$pcibus->{devices}},
3918     }
3919
3920     while (@$devices_to_check) {
3921         my $to_check = [];
3922         for my $d (@$devices_to_check) {
3923             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3924             next if !$d->{'pci_bridge'};
3925
3926             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3927             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3928         }
3929         $devices_to_check = $to_check;
3930     }
3931
3932     my $resblock = vm_mon_cmd($vmid, 'query-block');
3933     foreach my $block (@$resblock) {
3934         if($block->{device} =~ m/^drive-(\S+)/){
3935                 $devices->{$1} = 1;
3936         }
3937     }
3938
3939     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3940     foreach my $mice (@$resmice) {
3941         if ($mice->{name} eq 'QEMU HID Tablet') {
3942             $devices->{tablet} = 1;
3943             last;
3944         }
3945     }
3946
3947     # for usb devices there is no query-usb
3948     # but we can iterate over the entries in
3949     # qom-list path=/machine/peripheral
3950     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3951     foreach my $per (@$resperipheral) {
3952         if ($per->{name} =~ m/^usb\d+$/) {
3953             $devices->{$per->{name}} = 1;
3954         }
3955     }
3956
3957     return $devices;
3958 }
3959
3960 sub vm_deviceplug {
3961     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3962
3963     my $q35 = machine_type_is_q35($conf);
3964
3965     my $devices_list = vm_devices_list($vmid);
3966     return 1 if defined($devices_list->{$deviceid});
3967
3968     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3969
3970     if ($deviceid eq 'tablet') {
3971
3972         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3973
3974     } elsif ($deviceid eq 'keyboard') {
3975
3976         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3977
3978     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3979
3980         die "usb hotplug currently not reliable\n";
3981         # since we can't reliably hot unplug all added usb devices
3982         # and usb passthrough disables live migration
3983         # we disable usb hotplugging for now
3984         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3985
3986     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3987
3988         qemu_iothread_add($vmid, $deviceid, $device);
3989
3990         qemu_driveadd($storecfg, $vmid, $device);
3991         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3992
3993         qemu_deviceadd($vmid, $devicefull);
3994         eval { qemu_deviceaddverify($vmid, $deviceid); };
3995         if (my $err = $@) {
3996             eval { qemu_drivedel($vmid, $deviceid); };
3997             warn $@ if $@;
3998             die $err;
3999         }
4000
4001     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4002
4003
4004         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4005         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4006         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4007
4008         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4009
4010         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4011             qemu_iothread_add($vmid, $deviceid, $device);
4012             $devicefull .= ",iothread=iothread-$deviceid";
4013         }
4014
4015         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4016             $devicefull .= ",num_queues=$device->{queues}";
4017         }
4018
4019         qemu_deviceadd($vmid, $devicefull);
4020         qemu_deviceaddverify($vmid, $deviceid);
4021
4022     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4023
4024         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4025         qemu_driveadd($storecfg, $vmid, $device);
4026
4027         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4028         eval { qemu_deviceadd($vmid, $devicefull); };
4029         if (my $err = $@) {
4030             eval { qemu_drivedel($vmid, $deviceid); };
4031             warn $@ if $@;
4032             die $err;
4033         }
4034
4035     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4036
4037         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4038
4039         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4040         my $use_old_bios_files = undef;
4041         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4042
4043         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4044         qemu_deviceadd($vmid, $netdevicefull);
4045         eval { qemu_deviceaddverify($vmid, $deviceid); };
4046         if (my $err = $@) {
4047             eval { qemu_netdevdel($vmid, $deviceid); };
4048             warn $@ if $@;
4049             die $err;
4050         }
4051
4052     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4053
4054         my $bridgeid = $2;
4055         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4056         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4057
4058         qemu_deviceadd($vmid, $devicefull);
4059         qemu_deviceaddverify($vmid, $deviceid);
4060
4061     } else {
4062         die "can't hotplug device '$deviceid'\n";
4063     }
4064
4065     return 1;
4066 }
4067
4068 # fixme: this should raise exceptions on error!
4069 sub vm_deviceunplug {
4070     my ($vmid, $conf, $deviceid) = @_;
4071
4072     my $devices_list = vm_devices_list($vmid);
4073     return 1 if !defined($devices_list->{$deviceid});
4074
4075     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4076
4077     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4078
4079         qemu_devicedel($vmid, $deviceid);
4080
4081     } elsif ($deviceid =~ m/^usb\d+$/) {
4082
4083         die "usb hotplug currently not reliable\n";
4084         # when unplugging usb devices this way,
4085         # there may be remaining usb controllers/hubs
4086         # so we disable it for now
4087         qemu_devicedel($vmid, $deviceid);
4088         qemu_devicedelverify($vmid, $deviceid);
4089
4090     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4091
4092         qemu_devicedel($vmid, $deviceid);
4093         qemu_devicedelverify($vmid, $deviceid);
4094         qemu_drivedel($vmid, $deviceid);
4095         qemu_iothread_del($conf, $vmid, $deviceid);
4096
4097     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4098
4099         qemu_devicedel($vmid, $deviceid);
4100         qemu_devicedelverify($vmid, $deviceid);
4101         qemu_iothread_del($conf, $vmid, $deviceid);
4102
4103     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4104
4105         qemu_devicedel($vmid, $deviceid);
4106         qemu_drivedel($vmid, $deviceid);
4107         qemu_deletescsihw($conf, $vmid, $deviceid);
4108
4109     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4110
4111         qemu_devicedel($vmid, $deviceid);
4112         qemu_devicedelverify($vmid, $deviceid);
4113         qemu_netdevdel($vmid, $deviceid);
4114
4115     } else {
4116         die "can't unplug device '$deviceid'\n";
4117     }
4118
4119     return 1;
4120 }
4121
4122 sub qemu_deviceadd {
4123     my ($vmid, $devicefull) = @_;
4124
4125     $devicefull = "driver=".$devicefull;
4126     my %options =  split(/[=,]/, $devicefull);
4127
4128     vm_mon_cmd($vmid, "device_add" , %options);
4129 }
4130
4131 sub qemu_devicedel {
4132     my ($vmid, $deviceid) = @_;
4133
4134     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4135 }
4136
4137 sub qemu_iothread_add {
4138     my($vmid, $deviceid, $device) = @_;
4139
4140     if ($device->{iothread}) {
4141         my $iothreads = vm_iothreads_list($vmid);
4142         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4143     }
4144 }
4145
4146 sub qemu_iothread_del {
4147     my($conf, $vmid, $deviceid) = @_;
4148
4149     my $device = parse_drive($deviceid, $conf->{$deviceid});
4150     if ($device->{iothread}) {
4151         my $iothreads = vm_iothreads_list($vmid);
4152         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4153     }
4154 }
4155
4156 sub qemu_objectadd {
4157     my($vmid, $objectid, $qomtype) = @_;
4158
4159     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4160
4161     return 1;
4162 }
4163
4164 sub qemu_objectdel {
4165     my($vmid, $objectid) = @_;
4166
4167     vm_mon_cmd($vmid, "object-del", id => $objectid);
4168
4169     return 1;
4170 }
4171
4172 sub qemu_driveadd {
4173     my ($storecfg, $vmid, $device) = @_;
4174
4175     my $drive = print_drive_full($storecfg, $vmid, $device);
4176     $drive =~ s/\\/\\\\/g;
4177     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4178
4179     # If the command succeeds qemu prints: "OK"
4180     return 1 if $ret =~ m/OK/s;
4181
4182     die "adding drive failed: $ret\n";
4183 }
4184
4185 sub qemu_drivedel {
4186     my($vmid, $deviceid) = @_;
4187
4188     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4189     $ret =~ s/^\s+//;
4190
4191     return 1 if $ret eq "";
4192
4193     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4194     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4195
4196     die "deleting drive $deviceid failed : $ret\n";
4197 }
4198
4199 sub qemu_deviceaddverify {
4200     my ($vmid, $deviceid) = @_;
4201
4202     for (my $i = 0; $i <= 5; $i++) {
4203          my $devices_list = vm_devices_list($vmid);
4204          return 1 if defined($devices_list->{$deviceid});
4205          sleep 1;
4206     }
4207
4208     die "error on hotplug device '$deviceid'\n";
4209 }
4210
4211
4212 sub qemu_devicedelverify {
4213     my ($vmid, $deviceid) = @_;
4214
4215     # need to verify that the device is correctly removed as device_del
4216     # is async and empty return is not reliable
4217
4218     for (my $i = 0; $i <= 5; $i++) {
4219          my $devices_list = vm_devices_list($vmid);
4220          return 1 if !defined($devices_list->{$deviceid});
4221          sleep 1;
4222     }
4223
4224     die "error on hot-unplugging device '$deviceid'\n";
4225 }
4226
4227 sub qemu_findorcreatescsihw {
4228     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4229
4230     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4231
4232     my $scsihwid="$controller_prefix$controller";
4233     my $devices_list = vm_devices_list($vmid);
4234
4235     if(!defined($devices_list->{$scsihwid})) {
4236         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4237     }
4238
4239     return 1;
4240 }
4241
4242 sub qemu_deletescsihw {
4243     my ($conf, $vmid, $opt) = @_;
4244
4245     my $device = parse_drive($opt, $conf->{$opt});
4246
4247     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4248         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4249         return 1;
4250     }
4251
4252     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4253
4254     my $devices_list = vm_devices_list($vmid);
4255     foreach my $opt (keys %{$devices_list}) {
4256         if (PVE::QemuServer::is_valid_drivename($opt)) {
4257             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4258             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4259                 return 1;
4260             }
4261         }
4262     }
4263
4264     my $scsihwid="scsihw$controller";
4265
4266     vm_deviceunplug($vmid, $conf, $scsihwid);
4267
4268     return 1;
4269 }
4270
4271 sub qemu_add_pci_bridge {
4272     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4273
4274     my $bridges = {};
4275
4276     my $bridgeid;
4277
4278     print_pci_addr($device, $bridges, $arch, $machine_type);
4279
4280     while (my ($k, $v) = each %$bridges) {
4281         $bridgeid = $k;
4282     }
4283     return 1 if !defined($bridgeid) || $bridgeid < 1;
4284
4285     my $bridge = "pci.$bridgeid";
4286     my $devices_list = vm_devices_list($vmid);
4287
4288     if (!defined($devices_list->{$bridge})) {
4289         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4290     }
4291
4292     return 1;
4293 }
4294
4295 sub qemu_set_link_status {
4296     my ($vmid, $device, $up) = @_;
4297
4298     vm_mon_cmd($vmid, "set_link", name => $device,
4299                up => $up ? JSON::true : JSON::false);
4300 }
4301
4302 sub qemu_netdevadd {
4303     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4304
4305     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4306     my %options =  split(/[=,]/, $netdev);
4307
4308     vm_mon_cmd($vmid, "netdev_add",  %options);
4309     return 1;
4310 }
4311
4312 sub qemu_netdevdel {
4313     my ($vmid, $deviceid) = @_;
4314
4315     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4316 }
4317
4318 sub qemu_usb_hotplug {
4319     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4320
4321     return if !$device;
4322
4323     # remove the old one first
4324     vm_deviceunplug($vmid, $conf, $deviceid);
4325
4326     # check if xhci controller is necessary and available
4327     if ($device->{usb3}) {
4328
4329         my $devicelist = vm_devices_list($vmid);
4330
4331         if (!$devicelist->{xhci}) {
4332             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4333             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4334         }
4335     }
4336     my $d = parse_usb_device($device->{host});
4337     $d->{usb3} = $device->{usb3};
4338
4339     # add the new one
4340     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4341 }
4342
4343 sub qemu_cpu_hotplug {
4344     my ($vmid, $conf, $vcpus) = @_;
4345
4346     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4347
4348     my $sockets = 1;
4349     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4350     $sockets = $conf->{sockets} if  $conf->{sockets};
4351     my $cores = $conf->{cores} || 1;
4352     my $maxcpus = $sockets * $cores;
4353
4354     $vcpus = $maxcpus if !$vcpus;
4355
4356     die "you can't add more vcpus than maxcpus\n"
4357         if $vcpus > $maxcpus;
4358
4359     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4360
4361     if ($vcpus < $currentvcpus) {
4362
4363         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4364
4365             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4366                 qemu_devicedel($vmid, "cpu$i");
4367                 my $retry = 0;
4368                 my $currentrunningvcpus = undef;
4369                 while (1) {
4370                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4371                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4372                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4373                     $retry++;
4374                     sleep 1;
4375                 }
4376                 #update conf after each succesfull cpu unplug
4377                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4378                 PVE::QemuConfig->write_config($vmid, $conf);
4379             }
4380         } else {
4381             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4382         }
4383
4384         return;
4385     }
4386
4387     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4388     die "vcpus in running vm does not match its configuration\n"
4389         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4390
4391     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4392
4393         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4394             my $cpustr = print_cpu_device($conf, $i);
4395             qemu_deviceadd($vmid, $cpustr);
4396
4397             my $retry = 0;
4398             my $currentrunningvcpus = undef;
4399             while (1) {
4400                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4401                 last if scalar(@{$currentrunningvcpus}) == $i;
4402                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4403                 sleep 1;
4404                 $retry++;
4405             }
4406             #update conf after each succesfull cpu hotplug
4407             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4408             PVE::QemuConfig->write_config($vmid, $conf);
4409         }
4410     } else {
4411
4412         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4413             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4414         }
4415     }
4416 }
4417
4418 sub qemu_block_set_io_throttle {
4419     my ($vmid, $deviceid,
4420         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4421         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4422         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4423         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4424
4425     return if !check_running($vmid) ;
4426
4427     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4428         bps => int($bps),
4429         bps_rd => int($bps_rd),
4430         bps_wr => int($bps_wr),
4431         iops => int($iops),
4432         iops_rd => int($iops_rd),
4433         iops_wr => int($iops_wr),
4434         bps_max => int($bps_max),
4435         bps_rd_max => int($bps_rd_max),
4436         bps_wr_max => int($bps_wr_max),
4437         iops_max => int($iops_max),
4438         iops_rd_max => int($iops_rd_max),
4439         iops_wr_max => int($iops_wr_max),
4440         bps_max_length => int($bps_max_length),
4441         bps_rd_max_length => int($bps_rd_max_length),
4442         bps_wr_max_length => int($bps_wr_max_length),
4443         iops_max_length => int($iops_max_length),
4444         iops_rd_max_length => int($iops_rd_max_length),
4445         iops_wr_max_length => int($iops_wr_max_length),
4446     );
4447
4448 }
4449
4450 # old code, only used to shutdown old VM after update
4451 sub __read_avail {
4452     my ($fh, $timeout) = @_;
4453
4454     my $sel = new IO::Select;
4455     $sel->add($fh);
4456
4457     my $res = '';
4458     my $buf;
4459
4460     my @ready;
4461     while (scalar (@ready = $sel->can_read($timeout))) {
4462         my $count;
4463         if ($count = $fh->sysread($buf, 8192)) {
4464             if ($buf =~ /^(.*)\(qemu\) $/s) {
4465                 $res .= $1;
4466                 last;
4467             } else {
4468                 $res .= $buf;
4469             }
4470         } else {
4471             if (!defined($count)) {
4472                 die "$!\n";
4473             }
4474             last;
4475         }
4476     }
4477
4478     die "monitor read timeout\n" if !scalar(@ready);
4479
4480     return $res;
4481 }
4482
4483 sub qemu_block_resize {
4484     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4485
4486     my $running = check_running($vmid);
4487
4488     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4489
4490     return if !$running;
4491
4492     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4493
4494 }
4495
4496 sub qemu_volume_snapshot {
4497     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4498
4499     my $running = check_running($vmid);
4500
4501     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4502         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4503     } else {
4504         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4505     }
4506 }
4507
4508 sub qemu_volume_snapshot_delete {
4509     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4510
4511     my $running = check_running($vmid);
4512
4513     if($running) {
4514
4515         $running = undef;
4516         my $conf = PVE::QemuConfig->load_config($vmid);
4517         foreach_drive($conf, sub {
4518             my ($ds, $drive) = @_;
4519             $running = 1 if $drive->{file} eq $volid;
4520         });
4521     }
4522
4523     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4524         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4525     } else {
4526         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4527     }
4528 }
4529
4530 sub set_migration_caps {
4531     my ($vmid) = @_;
4532
4533     my $cap_ref = [];
4534
4535     my $enabled_cap = {
4536         "auto-converge" => 1,
4537         "xbzrle" => 1,
4538         "x-rdma-pin-all" => 0,
4539         "zero-blocks" => 0,
4540         "compress" => 0
4541     };
4542
4543     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4544
4545     for my $supported_capability (@$supported_capabilities) {
4546         push @$cap_ref, {
4547             capability => $supported_capability->{capability},
4548             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4549         };
4550     }
4551
4552     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4553 }
4554
4555 my $fast_plug_option = {
4556     'lock' => 1,
4557     'name' => 1,
4558     'onboot' => 1,
4559     'shares' => 1,
4560     'startup' => 1,
4561     'description' => 1,
4562     'protection' => 1,
4563     'vmstatestorage' => 1,
4564 };
4565
4566 # hotplug changes in [PENDING]
4567 # $selection hash can be used to only apply specified options, for
4568 # example: { cores => 1 } (only apply changed 'cores')
4569 # $errors ref is used to return error messages
4570 sub vmconfig_hotplug_pending {
4571     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4572
4573     my $defaults = load_defaults();
4574     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4575
4576     # commit values which do not have any impact on running VM first
4577     # Note: those option cannot raise errors, we we do not care about
4578     # $selection and always apply them.
4579
4580     my $add_error = sub {
4581         my ($opt, $msg) = @_;
4582         $errors->{$opt} = "hotplug problem - $msg";
4583     };
4584
4585     my $changes = 0;
4586     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4587         if ($fast_plug_option->{$opt}) {
4588             $conf->{$opt} = $conf->{pending}->{$opt};
4589             delete $conf->{pending}->{$opt};
4590             $changes = 1;
4591         }
4592     }
4593
4594     if ($changes) {
4595         PVE::QemuConfig->write_config($vmid, $conf);
4596         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4597     }
4598
4599     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4600
4601     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4602     while (my ($opt, $force) = each %$pending_delete_hash) {
4603         next if $selection && !$selection->{$opt};
4604         eval {
4605             if ($opt eq 'hotplug') {
4606                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4607             } elsif ($opt eq 'tablet') {
4608                 die "skip\n" if !$hotplug_features->{usb};
4609                 if ($defaults->{tablet}) {
4610                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4611                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4612                         if $arch eq 'aarch64';
4613                 } else {
4614                     vm_deviceunplug($vmid, $conf, 'tablet');
4615                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4616                 }
4617             } elsif ($opt =~ m/^usb\d+/) {
4618                 die "skip\n";
4619                 # since we cannot reliably hot unplug usb devices
4620                 # we are disabling it
4621                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4622                 vm_deviceunplug($vmid, $conf, $opt);
4623             } elsif ($opt eq 'vcpus') {
4624                 die "skip\n" if !$hotplug_features->{cpu};
4625                 qemu_cpu_hotplug($vmid, $conf, undef);
4626             } elsif ($opt eq 'balloon') {
4627                 # enable balloon device is not hotpluggable
4628                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4629                 # here we reset the ballooning value to memory
4630                 my $balloon = $conf->{memory} || $defaults->{memory};
4631                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4632             } elsif ($fast_plug_option->{$opt}) {
4633                 # do nothing
4634             } elsif ($opt =~ m/^net(\d+)$/) {
4635                 die "skip\n" if !$hotplug_features->{network};
4636                 vm_deviceunplug($vmid, $conf, $opt);
4637             } elsif (is_valid_drivename($opt)) {
4638                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4639                 vm_deviceunplug($vmid, $conf, $opt);
4640                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4641             } elsif ($opt =~ m/^memory$/) {
4642                 die "skip\n" if !$hotplug_features->{memory};
4643                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4644             } elsif ($opt eq 'cpuunits') {
4645                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4646             } elsif ($opt eq 'cpulimit') {
4647                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4648             } else {
4649                 die "skip\n";
4650             }
4651         };
4652         if (my $err = $@) {
4653             &$add_error($opt, $err) if $err ne "skip\n";
4654         } else {
4655             # save new config if hotplug was successful
4656             delete $conf->{$opt};
4657             vmconfig_undelete_pending_option($conf, $opt);
4658             PVE::QemuConfig->write_config($vmid, $conf);
4659             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4660         }
4661     }
4662
4663     my $apply_pending_cloudinit;
4664     $apply_pending_cloudinit = sub {
4665         my ($key, $value) = @_;
4666         $apply_pending_cloudinit = sub {}; # once is enough
4667
4668         my @cloudinit_opts = keys %$confdesc_cloudinit;
4669         foreach my $opt (keys %{$conf->{pending}}) {
4670             next if !grep { $_ eq $opt } @cloudinit_opts;
4671             $conf->{$opt} = delete $conf->{pending}->{$opt};
4672         }
4673
4674         my $new_conf = { %$conf };
4675         $new_conf->{$key} = $value;
4676         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4677     };
4678
4679     foreach my $opt (keys %{$conf->{pending}}) {
4680         next if $selection && !$selection->{$opt};
4681         my $value = $conf->{pending}->{$opt};
4682         eval {
4683             if ($opt eq 'hotplug') {
4684                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4685             } elsif ($opt eq 'tablet') {
4686                 die "skip\n" if !$hotplug_features->{usb};
4687                 if ($value == 1) {
4688                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4689                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4690                         if $arch eq 'aarch64';
4691                 } elsif ($value == 0) {
4692                     vm_deviceunplug($vmid, $conf, 'tablet');
4693                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4694                 }
4695             } elsif ($opt =~ m/^usb\d+$/) {
4696                 die "skip\n";
4697                 # since we cannot reliably hot unplug usb devices
4698                 # we are disabling it
4699                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4700                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4701                 die "skip\n" if !$d;
4702                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4703             } elsif ($opt eq 'vcpus') {
4704                 die "skip\n" if !$hotplug_features->{cpu};
4705                 qemu_cpu_hotplug($vmid, $conf, $value);
4706             } elsif ($opt eq 'balloon') {
4707                 # enable/disable balloning device is not hotpluggable
4708                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4709                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4710                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4711
4712                 # allow manual ballooning if shares is set to zero
4713                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4714                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4715                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4716                 }
4717             } elsif ($opt =~ m/^net(\d+)$/) {
4718                 # some changes can be done without hotplug
4719                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4720                                     $vmid, $opt, $value, $arch, $machine_type);
4721             } elsif (is_valid_drivename($opt)) {
4722                 # some changes can be done without hotplug
4723                 my $drive = parse_drive($opt, $value);
4724                 if (drive_is_cloudinit($drive)) {
4725                     &$apply_pending_cloudinit($opt, $value);
4726                 }
4727                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4728                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4729             } elsif ($opt =~ m/^memory$/) { #dimms
4730                 die "skip\n" if !$hotplug_features->{memory};
4731                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4732             } elsif ($opt eq 'cpuunits') {
4733                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4734             } elsif ($opt eq 'cpulimit') {
4735                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4736                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4737             } else {
4738                 die "skip\n";  # skip non-hot-pluggable options
4739             }
4740         };
4741         if (my $err = $@) {
4742             &$add_error($opt, $err) if $err ne "skip\n";
4743         } else {
4744             # save new config if hotplug was successful
4745             $conf->{$opt} = $value;
4746             delete $conf->{pending}->{$opt};
4747             PVE::QemuConfig->write_config($vmid, $conf);
4748             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4749         }
4750     }
4751 }
4752
4753 sub try_deallocate_drive {
4754     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4755
4756     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4757         my $volid = $drive->{file};
4758         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4759             my $sid = PVE::Storage::parse_volume_id($volid);
4760             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4761
4762             # check if the disk is really unused
4763             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4764                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4765             PVE::Storage::vdisk_free($storecfg, $volid);
4766             return 1;
4767         } else {
4768             # If vm is not owner of this disk remove from config
4769             return 1;
4770         }
4771     }
4772
4773     return undef;
4774 }
4775
4776 sub vmconfig_delete_or_detach_drive {
4777     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4778
4779     my $drive = parse_drive($opt, $conf->{$opt});
4780
4781     my $rpcenv = PVE::RPCEnvironment::get();
4782     my $authuser = $rpcenv->get_user();
4783
4784     if ($force) {
4785         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4786         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4787     } else {
4788         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4789     }
4790 }
4791
4792 sub vmconfig_apply_pending {
4793     my ($vmid, $conf, $storecfg) = @_;
4794
4795     # cold plug
4796
4797     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4798     while (my ($opt, $force) = each %$pending_delete_hash) {
4799         die "internal error" if $opt =~ m/^unused/;
4800         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4801         if (!defined($conf->{$opt})) {
4802             vmconfig_undelete_pending_option($conf, $opt);
4803             PVE::QemuConfig->write_config($vmid, $conf);
4804         } elsif (is_valid_drivename($opt)) {
4805             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4806             vmconfig_undelete_pending_option($conf, $opt);
4807             delete $conf->{$opt};
4808             PVE::QemuConfig->write_config($vmid, $conf);
4809         } else {
4810             vmconfig_undelete_pending_option($conf, $opt);
4811             delete $conf->{$opt};
4812             PVE::QemuConfig->write_config($vmid, $conf);
4813         }
4814     }
4815
4816     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4817
4818     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4819         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4820
4821         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4822             # skip if nothing changed
4823         } elsif (is_valid_drivename($opt)) {
4824             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4825                 if defined($conf->{$opt});
4826             $conf->{$opt} = $conf->{pending}->{$opt};
4827         } else {
4828             $conf->{$opt} = $conf->{pending}->{$opt};
4829         }
4830
4831         delete $conf->{pending}->{$opt};
4832         PVE::QemuConfig->write_config($vmid, $conf);
4833     }
4834 }
4835
4836 my $safe_num_ne = sub {
4837     my ($a, $b) = @_;
4838
4839     return 0 if !defined($a) && !defined($b);
4840     return 1 if !defined($a);
4841     return 1 if !defined($b);
4842
4843     return $a != $b;
4844 };
4845
4846 my $safe_string_ne = sub {
4847     my ($a, $b) = @_;
4848
4849     return 0 if !defined($a) && !defined($b);
4850     return 1 if !defined($a);
4851     return 1 if !defined($b);
4852
4853     return $a ne $b;
4854 };
4855
4856 sub vmconfig_update_net {
4857     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4858
4859     my $newnet = parse_net($value);
4860
4861     if ($conf->{$opt}) {
4862         my $oldnet = parse_net($conf->{$opt});
4863
4864         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4865             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4866             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4867             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4868
4869             # for non online change, we try to hot-unplug
4870             die "skip\n" if !$hotplug;
4871             vm_deviceunplug($vmid, $conf, $opt);
4872         } else {
4873
4874             die "internal error" if $opt !~ m/net(\d+)/;
4875             my $iface = "tap${vmid}i$1";
4876
4877             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4878                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4879                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4880                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4881                 PVE::Network::tap_unplug($iface);
4882                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4883             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4884                 # Rate can be applied on its own but any change above needs to
4885                 # include the rate in tap_plug since OVS resets everything.
4886                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4887             }
4888
4889             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4890                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4891             }
4892
4893             return 1;
4894         }
4895     }
4896
4897     if ($hotplug) {
4898         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4899     } else {
4900         die "skip\n";
4901     }
4902 }
4903
4904 sub vmconfig_update_disk {
4905     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4906
4907     # fixme: do we need force?
4908
4909     my $drive = parse_drive($opt, $value);
4910
4911     if ($conf->{$opt}) {
4912
4913         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4914
4915             my $media = $drive->{media} || 'disk';
4916             my $oldmedia = $old_drive->{media} || 'disk';
4917             die "unable to change media type\n" if $media ne $oldmedia;
4918
4919             if (!drive_is_cdrom($old_drive)) {
4920
4921                 if ($drive->{file} ne $old_drive->{file}) {
4922
4923                     die "skip\n" if !$hotplug;
4924
4925                     # unplug and register as unused
4926                     vm_deviceunplug($vmid, $conf, $opt);
4927                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4928
4929                 } else {
4930                     # update existing disk
4931
4932                     # skip non hotpluggable value
4933                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4934                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4935                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4936                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4937                         die "skip\n";
4938                     }
4939
4940                     # apply throttle
4941                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4942                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4943                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4944                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4945                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4946                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4947                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4948                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4949                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4950                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4951                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4952                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4953                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4954                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4955                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4956                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4957                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4958                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4959
4960                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4961                                                    ($drive->{mbps} || 0)*1024*1024,
4962                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4963                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4964                                                    $drive->{iops} || 0,
4965                                                    $drive->{iops_rd} || 0,
4966                                                    $drive->{iops_wr} || 0,
4967                                                    ($drive->{mbps_max} || 0)*1024*1024,
4968                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4969                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4970                                                    $drive->{iops_max} || 0,
4971                                                    $drive->{iops_rd_max} || 0,
4972                                                    $drive->{iops_wr_max} || 0,
4973                                                    $drive->{bps_max_length} || 1,
4974                                                    $drive->{bps_rd_max_length} || 1,
4975                                                    $drive->{bps_wr_max_length} || 1,
4976                                                    $drive->{iops_max_length} || 1,
4977                                                    $drive->{iops_rd_max_length} || 1,
4978                                                    $drive->{iops_wr_max_length} || 1);
4979
4980                     }
4981
4982                     return 1;
4983                 }
4984
4985             } else { # cdrom
4986
4987                 if ($drive->{file} eq 'none') {
4988                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4989                     if (drive_is_cloudinit($old_drive)) {
4990                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4991                     }
4992                 } else {
4993                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4994                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4995                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4996                 }
4997
4998                 return 1;
4999             }
5000         }
5001     }
5002
5003     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5004     # hotplug new disks
5005     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5006     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5007 }
5008
5009 sub vm_start {
5010     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5011         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5012
5013     PVE::QemuConfig->lock_config($vmid, sub {
5014         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5015
5016         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5017
5018         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5019
5020         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5021
5022         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5023             vmconfig_apply_pending($vmid, $conf, $storecfg);
5024             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5025         }
5026
5027         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5028
5029         my $defaults = load_defaults();
5030
5031         # set environment variable useful inside network script
5032         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5033
5034         my $local_volumes = {};
5035
5036         if ($targetstorage) {
5037             foreach_drive($conf, sub {
5038                 my ($ds, $drive) = @_;
5039
5040                 return if drive_is_cdrom($drive);
5041
5042                 my $volid = $drive->{file};
5043
5044                 return if !$volid;
5045
5046                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5047
5048                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5049                 return if $scfg->{shared};
5050                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5051             });
5052
5053             my $format = undef;
5054
5055             foreach my $opt (sort keys %$local_volumes) {
5056
5057                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5058                 my $drive = parse_drive($opt, $conf->{$opt});
5059
5060                 #if remote storage is specified, use default format
5061                 if ($targetstorage && $targetstorage ne "1") {
5062                     $storeid = $targetstorage;
5063                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5064                     $format = $defFormat;
5065                 } else {
5066                     #else we use same format than original
5067                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5068                     $format = qemu_img_format($scfg, $volid);
5069                 }
5070
5071                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5072                 my $newdrive = $drive;
5073                 $newdrive->{format} = $format;
5074                 $newdrive->{file} = $newvolid;
5075                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5076                 $local_volumes->{$opt} = $drivestr;
5077                 #pass drive to conf for command line
5078                 $conf->{$opt} = $drivestr;
5079             }
5080         }
5081
5082         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5083
5084         my $migrate_port = 0;
5085         my $migrate_uri;
5086         if ($statefile) {
5087             if ($statefile eq 'tcp') {
5088                 my $localip = "localhost";
5089                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5090                 my $nodename = PVE::INotify::nodename();
5091
5092                 if (!defined($migration_type)) {
5093                     if (defined($datacenterconf->{migration}->{type})) {
5094                         $migration_type = $datacenterconf->{migration}->{type};
5095                     } else {
5096                         $migration_type = 'secure';
5097                     }
5098                 }
5099
5100                 if ($migration_type eq 'insecure') {
5101                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5102                     if ($migrate_network_addr) {
5103                         $localip = $migrate_network_addr;
5104                     } else {
5105                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5106                     }
5107
5108                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5109                 }
5110
5111                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5112                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5113                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5114                 push @$cmd, '-incoming', $migrate_uri;
5115                 push @$cmd, '-S';
5116
5117             } elsif ($statefile eq 'unix') {
5118                 # should be default for secure migrations as a ssh TCP forward
5119                 # tunnel is not deterministic reliable ready and fails regurarly
5120                 # to set up in time, so use UNIX socket forwards
5121                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5122                 unlink $socket_addr;
5123
5124                 $migrate_uri = "unix:$socket_addr";
5125
5126                 push @$cmd, '-incoming', $migrate_uri;
5127                 push @$cmd, '-S';
5128
5129             } else {
5130                 push @$cmd, '-loadstate', $statefile;
5131             }
5132         } elsif ($paused) {
5133             push @$cmd, '-S';
5134         }
5135
5136         # host pci devices
5137         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5138           my $d = parse_hostpci($conf->{"hostpci$i"});
5139           next if !$d;
5140           my $pcidevices = $d->{pciid};
5141           foreach my $pcidevice (@$pcidevices) {
5142                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5143
5144                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5145                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5146                 die "no pci device info for device '$pciid'\n" if !$info;
5147                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5148                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5149                 die "can't reset pci device '$pciid'\n"
5150                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5151           }
5152         }
5153
5154         PVE::Storage::activate_volumes($storecfg, $vollist);
5155
5156         if (!check_running($vmid, 1)) {
5157             eval {
5158                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5159                     outfunc => sub {}, errfunc => sub {});
5160             };
5161         }
5162
5163         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5164                                                   : $defaults->{cpuunits};
5165
5166         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5167         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5168
5169         my %properties = (
5170             Slice => 'qemu.slice',
5171             KillMode => 'none',
5172             CPUShares => $cpuunits
5173         );
5174
5175         if (my $cpulimit = $conf->{cpulimit}) {
5176             $properties{CPUQuota} = int($cpulimit * 100);
5177         }
5178         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5179
5180         my $run_qemu = sub {
5181             PVE::Tools::run_fork sub {
5182                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5183                 run_command($cmd, %run_params);
5184             };
5185         };
5186
5187         if ($conf->{hugepages}) {
5188
5189             my $code = sub {
5190                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5191                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5192
5193                 PVE::QemuServer::Memory::hugepages_mount();
5194                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5195
5196                 eval { $run_qemu->() };
5197                 if (my $err = $@) {
5198                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5199                     die $err;
5200                 }
5201
5202                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5203             };
5204             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5205
5206         } else {
5207             eval { $run_qemu->() };
5208         }
5209
5210         if (my $err = $@) {
5211             # deactivate volumes if start fails
5212             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5213             die "start failed: $err";
5214         }
5215
5216         print "migration listens on $migrate_uri\n" if $migrate_uri;
5217
5218         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5219             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5220             warn $@ if $@;
5221         }
5222
5223         #start nbd server for storage migration
5224         if ($targetstorage) {
5225             my $nodename = PVE::INotify::nodename();
5226             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5227             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5228             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5229             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5230
5231             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5232
5233             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5234
5235             foreach my $opt (sort keys %$local_volumes) {
5236                 my $volid = $local_volumes->{$opt};
5237                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5238                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5239                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5240             }
5241         }
5242
5243         if ($migratedfrom) {
5244             eval {
5245                 set_migration_caps($vmid);
5246             };
5247             warn $@ if $@;
5248
5249             if ($spice_port) {
5250                 print "spice listens on port $spice_port\n";
5251                 if ($spice_ticket) {
5252                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5253                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5254                 }
5255             }
5256
5257         } else {
5258             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5259                 if !$statefile && $conf->{balloon};
5260
5261             foreach my $opt (keys %$conf) {
5262                 next if $opt !~  m/^net\d+$/;
5263                 my $nicconf = parse_net($conf->{$opt});
5264                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5265             }
5266         }
5267
5268         vm_mon_cmd_nocheck($vmid, 'qom-set',
5269                     path => "machine/peripheral/balloon0",
5270                     property => "guest-stats-polling-interval",
5271                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5272
5273     });
5274 }
5275
5276 sub vm_mon_cmd {
5277     my ($vmid, $execute, %params) = @_;
5278
5279     my $cmd = { execute => $execute, arguments => \%params };
5280     vm_qmp_command($vmid, $cmd);
5281 }
5282
5283 sub vm_mon_cmd_nocheck {
5284     my ($vmid, $execute, %params) = @_;
5285
5286     my $cmd = { execute => $execute, arguments => \%params };
5287     vm_qmp_command($vmid, $cmd, 1);
5288 }
5289
5290 sub vm_qmp_command {
5291     my ($vmid, $cmd, $nocheck) = @_;
5292
5293     my $res;
5294
5295     my $timeout;
5296     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5297         $timeout = $cmd->{arguments}->{timeout};
5298         delete $cmd->{arguments}->{timeout};
5299     }
5300
5301     eval {
5302         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5303         my $sname = qmp_socket($vmid);
5304         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5305             my $qmpclient = PVE::QMPClient->new();
5306
5307             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5308         } else {
5309             die "unable to open monitor socket\n";
5310         }
5311     };
5312     if (my $err = $@) {
5313         syslog("err", "VM $vmid qmp command failed - $err");
5314         die $err;
5315     }
5316
5317     return $res;
5318 }
5319
5320 sub vm_human_monitor_command {
5321     my ($vmid, $cmdline) = @_;
5322
5323     my $res;
5324
5325     my $cmd = {
5326         execute => 'human-monitor-command',
5327         arguments => { 'command-line' => $cmdline},
5328     };
5329
5330     return vm_qmp_command($vmid, $cmd);
5331 }
5332
5333 sub vm_commandline {
5334     my ($storecfg, $vmid) = @_;
5335
5336     my $conf = PVE::QemuConfig->load_config($vmid);
5337
5338     my $defaults = load_defaults();
5339
5340     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5341
5342     return PVE::Tools::cmd2string($cmd);
5343 }
5344
5345 sub vm_reset {
5346     my ($vmid, $skiplock) = @_;
5347
5348     PVE::QemuConfig->lock_config($vmid, sub {
5349
5350         my $conf = PVE::QemuConfig->load_config($vmid);
5351
5352         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5353
5354         vm_mon_cmd($vmid, "system_reset");
5355     });
5356 }
5357
5358 sub get_vm_volumes {
5359     my ($conf) = @_;
5360
5361     my $vollist = [];
5362     foreach_volid($conf, sub {
5363         my ($volid, $attr) = @_;
5364
5365         return if $volid =~ m|^/|;
5366
5367         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5368         return if !$sid;
5369
5370         push @$vollist, $volid;
5371     });
5372
5373     return $vollist;
5374 }
5375
5376 sub vm_stop_cleanup {
5377     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5378
5379     eval {
5380
5381         if (!$keepActive) {
5382             my $vollist = get_vm_volumes($conf);
5383             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5384         }
5385
5386         foreach my $ext (qw(mon qmp pid vnc qga)) {
5387             unlink "/var/run/qemu-server/${vmid}.$ext";
5388         }
5389
5390         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5391     };
5392     warn $@ if $@; # avoid errors - just warn
5393 }
5394
5395 # Note: use $nockeck to skip tests if VM configuration file exists.
5396 # We need that when migration VMs to other nodes (files already moved)
5397 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5398 sub vm_stop {
5399     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5400
5401     $force = 1 if !defined($force) && !$shutdown;
5402
5403     if ($migratedfrom){
5404         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5405         kill 15, $pid if $pid;
5406         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5407         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5408         return;
5409     }
5410
5411     PVE::QemuConfig->lock_config($vmid, sub {
5412
5413         my $pid = check_running($vmid, $nocheck);
5414         return if !$pid;
5415
5416         my $conf;
5417         if (!$nocheck) {
5418             $conf = PVE::QemuConfig->load_config($vmid);
5419             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5420             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5421                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5422                 $timeout = $opts->{down} if $opts->{down};
5423             }
5424         }
5425
5426         $timeout = 60 if !defined($timeout);
5427
5428         eval {
5429             if ($shutdown) {
5430                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5431                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5432                 } else {
5433                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5434                 }
5435             } else {
5436                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5437             }
5438         };
5439         my $err = $@;
5440
5441         if (!$err) {
5442             my $count = 0;
5443             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5444                 $count++;
5445                 sleep 1;
5446             }
5447
5448             if ($count >= $timeout) {
5449                 if ($force) {
5450                     warn "VM still running - terminating now with SIGTERM\n";
5451                     kill 15, $pid;
5452                 } else {
5453                     die "VM quit/powerdown failed - got timeout\n";
5454                 }
5455             } else {
5456                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5457                 return;
5458             }
5459         } else {
5460             if ($force) {
5461                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5462                 kill 15, $pid;
5463             } else {
5464                 die "VM quit/powerdown failed\n";
5465             }
5466         }
5467
5468         # wait again
5469         $timeout = 10;
5470
5471         my $count = 0;
5472         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5473             $count++;
5474             sleep 1;
5475         }
5476
5477         if ($count >= $timeout) {
5478             warn "VM still running - terminating now with SIGKILL\n";
5479             kill 9, $pid;
5480             sleep 1;
5481         }
5482
5483         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5484    });
5485 }
5486
5487 sub vm_suspend {
5488     my ($vmid, $skiplock) = @_;
5489
5490     PVE::QemuConfig->lock_config($vmid, sub {
5491
5492         my $conf = PVE::QemuConfig->load_config($vmid);
5493
5494         PVE::QemuConfig->check_lock($conf)
5495             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5496
5497         vm_mon_cmd($vmid, "stop");
5498     });
5499 }
5500
5501 sub vm_resume {
5502     my ($vmid, $skiplock, $nocheck) = @_;
5503
5504     PVE::QemuConfig->lock_config($vmid, sub {
5505
5506         my $res = vm_mon_cmd($vmid, 'query-status');
5507         my $resume_cmd = 'cont';
5508
5509         if ($res->{status} && $res->{status} eq 'suspended') {
5510             $resume_cmd = 'system_wakeup';
5511         }
5512
5513         if (!$nocheck) {
5514
5515             my $conf = PVE::QemuConfig->load_config($vmid);
5516
5517             PVE::QemuConfig->check_lock($conf)
5518                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5519
5520             vm_mon_cmd($vmid, $resume_cmd);
5521
5522         } else {
5523             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5524         }
5525     });
5526 }
5527
5528 sub vm_sendkey {
5529     my ($vmid, $skiplock, $key) = @_;
5530
5531     PVE::QemuConfig->lock_config($vmid, sub {
5532
5533         my $conf = PVE::QemuConfig->load_config($vmid);
5534
5535         # there is no qmp command, so we use the human monitor command
5536         vm_human_monitor_command($vmid, "sendkey $key");
5537     });
5538 }
5539
5540 sub vm_destroy {
5541     my ($storecfg, $vmid, $skiplock) = @_;
5542
5543     PVE::QemuConfig->lock_config($vmid, sub {
5544
5545         my $conf = PVE::QemuConfig->load_config($vmid);
5546
5547         if (!check_running($vmid)) {
5548             destroy_vm($storecfg, $vmid, undef, $skiplock);
5549         } else {
5550             die "VM $vmid is running - destroy failed\n";
5551         }
5552     });
5553 }
5554
5555 # vzdump restore implementaion
5556
5557 sub tar_archive_read_firstfile {
5558     my $archive = shift;
5559
5560     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5561
5562     # try to detect archive type first
5563     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5564         die "unable to open file '$archive'\n";
5565     my $firstfile = <$fh>;
5566     kill 15, $pid;
5567     close $fh;
5568
5569     die "ERROR: archive contaions no data\n" if !$firstfile;
5570     chomp $firstfile;
5571
5572     return $firstfile;
5573 }
5574
5575 sub tar_restore_cleanup {
5576     my ($storecfg, $statfile) = @_;
5577
5578     print STDERR "starting cleanup\n";
5579
5580     if (my $fd = IO::File->new($statfile, "r")) {
5581         while (defined(my $line = <$fd>)) {
5582             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5583                 my $volid = $2;
5584                 eval {
5585                     if ($volid =~ m|^/|) {
5586                         unlink $volid || die 'unlink failed\n';
5587                     } else {
5588                         PVE::Storage::vdisk_free($storecfg, $volid);
5589                     }
5590                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5591                 };
5592                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5593             } else {
5594                 print STDERR "unable to parse line in statfile - $line";
5595             }
5596         }
5597         $fd->close();
5598     }
5599 }
5600
5601 sub restore_archive {
5602     my ($archive, $vmid, $user, $opts) = @_;
5603
5604     my $format = $opts->{format};
5605     my $comp;
5606
5607     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5608         $format = 'tar' if !$format;
5609         $comp = 'gzip';
5610     } elsif ($archive =~ m/\.tar$/) {
5611         $format = 'tar' if !$format;
5612     } elsif ($archive =~ m/.tar.lzo$/) {
5613         $format = 'tar' if !$format;
5614         $comp = 'lzop';
5615     } elsif ($archive =~ m/\.vma$/) {
5616         $format = 'vma' if !$format;
5617     } elsif ($archive =~ m/\.vma\.gz$/) {
5618         $format = 'vma' if !$format;
5619         $comp = 'gzip';
5620     } elsif ($archive =~ m/\.vma\.lzo$/) {
5621         $format = 'vma' if !$format;
5622         $comp = 'lzop';
5623     } else {
5624         $format = 'vma' if !$format; # default
5625     }
5626
5627     # try to detect archive format
5628     if ($format eq 'tar') {
5629         return restore_tar_archive($archive, $vmid, $user, $opts);
5630     } else {
5631         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5632     }
5633 }
5634
5635 sub restore_update_config_line {
5636     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5637
5638     return if $line =~ m/^\#qmdump\#/;
5639     return if $line =~ m/^\#vzdump\#/;
5640     return if $line =~ m/^lock:/;
5641     return if $line =~ m/^unused\d+:/;
5642     return if $line =~ m/^parent:/;
5643     return if $line =~ m/^template:/; # restored VM is never a template
5644
5645     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5646     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5647         # try to convert old 1.X settings
5648         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5649         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5650             my ($model, $macaddr) = split(/\=/, $devconfig);
5651             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5652             my $net = {
5653                 model => $model,
5654                 bridge => "vmbr$ind",
5655                 macaddr => $macaddr,
5656             };
5657             my $netstr = print_net($net);
5658
5659             print $outfd "net$cookie->{netcount}: $netstr\n";
5660             $cookie->{netcount}++;
5661         }
5662     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5663         my ($id, $netstr) = ($1, $2);
5664         my $net = parse_net($netstr);
5665         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5666         $netstr = print_net($net);
5667         print $outfd "$id: $netstr\n";
5668     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5669         my $virtdev = $1;
5670         my $value = $3;
5671         my $di = parse_drive($virtdev, $value);
5672         if (defined($di->{backup}) && !$di->{backup}) {
5673             print $outfd "#$line";
5674         } elsif ($map->{$virtdev}) {
5675             delete $di->{format}; # format can change on restore
5676             $di->{file} = $map->{$virtdev};
5677             $value = print_drive($vmid, $di);
5678             print $outfd "$virtdev: $value\n";
5679         } else {
5680             print $outfd $line;
5681         }
5682     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5683         my $vmgenid = $1;
5684         if ($vmgenid ne '0') {
5685             # always generate a new vmgenid if there was a valid one setup
5686             $vmgenid = generate_uuid();
5687         }
5688         print $outfd "vmgenid: $vmgenid\n";
5689     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5690         my ($uuid, $uuid_str);
5691         UUID::generate($uuid);
5692         UUID::unparse($uuid, $uuid_str);
5693         my $smbios1 = parse_smbios1($2);
5694         $smbios1->{uuid} = $uuid_str;
5695         print $outfd $1.print_smbios1($smbios1)."\n";
5696     } else {
5697         print $outfd $line;
5698     }
5699 }
5700
5701 sub scan_volids {
5702     my ($cfg, $vmid) = @_;
5703
5704     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5705
5706     my $volid_hash = {};
5707     foreach my $storeid (keys %$info) {
5708         foreach my $item (@{$info->{$storeid}}) {
5709             next if !($item->{volid} && $item->{size});
5710             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5711             $volid_hash->{$item->{volid}} = $item;
5712         }
5713     }
5714
5715     return $volid_hash;
5716 }
5717
5718 sub is_volume_in_use {
5719     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5720
5721     my $path = PVE::Storage::path($storecfg, $volid);
5722
5723     my $scan_config = sub {
5724         my ($cref, $snapname) = @_;
5725
5726         foreach my $key (keys %$cref) {
5727             my $value = $cref->{$key};
5728             if (is_valid_drivename($key)) {
5729                 next if $skip_drive && $key eq $skip_drive;
5730                 my $drive = parse_drive($key, $value);
5731                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5732                 return 1 if $volid eq $drive->{file};
5733                 if ($drive->{file} =~ m!^/!) {
5734                     return 1 if $drive->{file} eq $path;
5735                 } else {
5736                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5737                     next if !$storeid;
5738                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5739                     next if !$scfg;
5740                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5741                 }
5742             }
5743         }
5744
5745         return 0;
5746     };
5747
5748     return 1 if &$scan_config($conf);
5749
5750     undef $skip_drive;
5751
5752     foreach my $snapname (keys %{$conf->{snapshots}}) {
5753         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5754     }
5755
5756     return 0;
5757 }
5758
5759 sub update_disksize {
5760     my ($vmid, $conf, $volid_hash) = @_;
5761
5762     my $changes;
5763     my $prefix = "VM $vmid:";
5764
5765     # used and unused disks
5766     my $referenced = {};
5767
5768     # Note: it is allowed to define multiple storages with same path (alias), so
5769     # we need to check both 'volid' and real 'path' (two different volid can point
5770     # to the same path).
5771
5772     my $referencedpath = {};
5773
5774     # update size info
5775     foreach my $opt (keys %$conf) {
5776         if (is_valid_drivename($opt)) {
5777             my $drive = parse_drive($opt, $conf->{$opt});
5778             my $volid = $drive->{file};
5779             next if !$volid;
5780
5781             $referenced->{$volid} = 1;
5782             if ($volid_hash->{$volid} &&
5783                 (my $path = $volid_hash->{$volid}->{path})) {
5784                 $referencedpath->{$path} = 1;
5785             }
5786
5787             next if drive_is_cdrom($drive);
5788             next if !$volid_hash->{$volid};
5789
5790             $drive->{size} = $volid_hash->{$volid}->{size};
5791             my $new = print_drive($vmid, $drive);
5792             if ($new ne $conf->{$opt}) {
5793                 $changes = 1;
5794                 $conf->{$opt} = $new;
5795                 print "$prefix update disk '$opt' information.\n";
5796             }
5797         }
5798     }
5799
5800     # remove 'unusedX' entry if volume is used
5801     foreach my $opt (keys %$conf) {
5802         next if $opt !~ m/^unused\d+$/;
5803         my $volid = $conf->{$opt};
5804         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5805         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5806             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5807             $changes = 1;
5808             delete $conf->{$opt};
5809         }
5810
5811         $referenced->{$volid} = 1;
5812         $referencedpath->{$path} = 1 if $path;
5813     }
5814
5815     foreach my $volid (sort keys %$volid_hash) {
5816         next if $volid =~ m/vm-$vmid-state-/;
5817         next if $referenced->{$volid};
5818         my $path = $volid_hash->{$volid}->{path};
5819         next if !$path; # just to be sure
5820         next if $referencedpath->{$path};
5821         $changes = 1;
5822         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5823         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5824         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5825     }
5826
5827     return $changes;
5828 }
5829
5830 sub rescan {
5831     my ($vmid, $nolock, $dryrun) = @_;
5832
5833     my $cfg = PVE::Storage::config();
5834
5835     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5836     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5837     foreach my $stor (keys %{$cfg->{ids}}) {
5838         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5839     }
5840
5841     print "rescan volumes...\n";
5842     my $volid_hash = scan_volids($cfg, $vmid);
5843
5844     my $updatefn =  sub {
5845         my ($vmid) = @_;
5846
5847         my $conf = PVE::QemuConfig->load_config($vmid);
5848
5849         PVE::QemuConfig->check_lock($conf);
5850
5851         my $vm_volids = {};
5852         foreach my $volid (keys %$volid_hash) {
5853             my $info = $volid_hash->{$volid};
5854             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5855         }
5856
5857         my $changes = update_disksize($vmid, $conf, $vm_volids);
5858
5859         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5860     };
5861
5862     if (defined($vmid)) {
5863         if ($nolock) {
5864             &$updatefn($vmid);
5865         } else {
5866             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5867         }
5868     } else {
5869         my $vmlist = config_list();
5870         foreach my $vmid (keys %$vmlist) {
5871             if ($nolock) {
5872                 &$updatefn($vmid);
5873             } else {
5874                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5875             }
5876         }
5877     }
5878 }
5879
5880 sub restore_vma_archive {
5881     my ($archive, $vmid, $user, $opts, $comp) = @_;
5882
5883     my $readfrom = $archive;
5884
5885     my $cfg = PVE::Storage::config();
5886     my $commands = [];
5887     my $bwlimit = $opts->{bwlimit};
5888
5889     my $dbg_cmdstring = '';
5890     my $add_pipe = sub {
5891         my ($cmd) = @_;
5892         push @$commands, $cmd;
5893         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5894         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5895         $readfrom = '-';
5896     };
5897
5898     my $input = undef;
5899     if ($archive eq '-') {
5900         $input = '<&STDIN';
5901     } else {
5902         # If we use a backup from a PVE defined storage we also consider that
5903         # storage's rate limit:
5904         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5905         if (defined($volid)) {
5906             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5907             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5908             if ($readlimit) {
5909                 print STDERR "applying read rate limit: $readlimit\n";
5910                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5911                 $add_pipe->($cstream);
5912             }
5913         }
5914     }
5915
5916     if ($comp) {
5917         my $cmd;
5918         if ($comp eq 'gzip') {
5919             $cmd = ['zcat', $readfrom];
5920         } elsif ($comp eq 'lzop') {
5921             $cmd = ['lzop', '-d', '-c', $readfrom];
5922         } else {
5923             die "unknown compression method '$comp'\n";
5924         }
5925         $add_pipe->($cmd);
5926     }
5927
5928     my $tmpdir = "/var/tmp/vzdumptmp$$";
5929     rmtree $tmpdir;
5930
5931     # disable interrupts (always do cleanups)
5932     local $SIG{INT} =
5933         local $SIG{TERM} =
5934         local $SIG{QUIT} =
5935         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5936
5937     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5938     POSIX::mkfifo($mapfifo, 0600);
5939     my $fifofh;
5940
5941     my $openfifo = sub {
5942         open($fifofh, '>', $mapfifo) || die $!;
5943     };
5944
5945     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5946
5947     my $oldtimeout;
5948     my $timeout = 5;
5949
5950     my $devinfo = {};
5951
5952     my $rpcenv = PVE::RPCEnvironment::get();
5953
5954     my $conffile = PVE::QemuConfig->config_file($vmid);
5955     my $tmpfn = "$conffile.$$.tmp";
5956
5957     # Note: $oldconf is undef if VM does not exists
5958     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5959     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5960
5961     my %storage_limits;
5962
5963     my $print_devmap = sub {
5964         my $virtdev_hash = {};
5965
5966         my $cfgfn = "$tmpdir/qemu-server.conf";
5967
5968         # we can read the config - that is already extracted
5969         my $fh = IO::File->new($cfgfn, "r") ||
5970             "unable to read qemu-server.conf - $!\n";
5971
5972         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5973         if (-f $fwcfgfn) {
5974             my $pve_firewall_dir = '/etc/pve/firewall';
5975             mkdir $pve_firewall_dir; # make sure the dir exists
5976             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5977         }
5978
5979         while (defined(my $line = <$fh>)) {
5980             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5981                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5982                 die "archive does not contain data for drive '$virtdev'\n"
5983                     if !$devinfo->{$devname};
5984                 if (defined($opts->{storage})) {
5985                     $storeid = $opts->{storage} || 'local';
5986                 } elsif (!$storeid) {
5987                     $storeid = 'local';
5988                 }
5989                 $format = 'raw' if !$format;
5990                 $devinfo->{$devname}->{devname} = $devname;
5991                 $devinfo->{$devname}->{virtdev} = $virtdev;
5992                 $devinfo->{$devname}->{format} = $format;
5993                 $devinfo->{$devname}->{storeid} = $storeid;
5994
5995                 # check permission on storage
5996                 my $pool = $opts->{pool}; # todo: do we need that?
5997                 if ($user ne 'root@pam') {
5998                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5999                 }
6000
6001                 $storage_limits{$storeid} = $bwlimit;
6002
6003                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6004             }
6005         }
6006
6007         foreach my $key (keys %storage_limits) {
6008             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6009             next if !$limit;
6010             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6011             $storage_limits{$key} = $limit * 1024;
6012         }
6013
6014         foreach my $devname (keys %$devinfo) {
6015             die "found no device mapping information for device '$devname'\n"
6016                 if !$devinfo->{$devname}->{virtdev};
6017         }
6018
6019         # create empty/temp config
6020         if ($oldconf) {
6021             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6022             foreach_drive($oldconf, sub {
6023                 my ($ds, $drive) = @_;
6024
6025                 return if drive_is_cdrom($drive);
6026
6027                 my $volid = $drive->{file};
6028
6029                 return if !$volid || $volid =~ m|^/|;
6030
6031                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6032                 return if !$path || !$owner || ($owner != $vmid);
6033
6034                 # Note: only delete disk we want to restore
6035                 # other volumes will become unused
6036                 if ($virtdev_hash->{$ds}) {
6037                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6038                     if (my $err = $@) {
6039                         warn $err;
6040                     }
6041                 }
6042             });
6043
6044             # delete vmstate files
6045             # since after the restore we have no snapshots anymore
6046             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6047                 my $snap = $oldconf->{snapshots}->{$snapname};
6048                 if ($snap->{vmstate}) {
6049                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6050                     if (my $err = $@) {
6051                         warn $err;
6052                     }
6053                 }
6054             }
6055         }
6056
6057         my $map = {};
6058         foreach my $virtdev (sort keys %$virtdev_hash) {
6059             my $d = $virtdev_hash->{$virtdev};
6060             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6061             my $storeid = $d->{storeid};
6062             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6063
6064             my $map_opts = '';
6065             if (my $limit = $storage_limits{$storeid}) {
6066                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6067             }
6068
6069             # test if requested format is supported
6070             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6071             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6072             $d->{format} = $defFormat if !$supported;
6073
6074             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6075                                                   $d->{format}, undef, $alloc_size);
6076             print STDERR "new volume ID is '$volid'\n";
6077             $d->{volid} = $volid;
6078             my $path = PVE::Storage::path($cfg, $volid);
6079
6080             PVE::Storage::activate_volumes($cfg,[$volid]);
6081
6082             my $write_zeros = 1;
6083             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6084                 $write_zeros = 0;
6085             }
6086
6087             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6088
6089             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6090             $map->{$virtdev} = $volid;
6091         }
6092
6093         $fh->seek(0, 0) || die "seek failed - $!\n";
6094
6095         my $outfd = new IO::File ($tmpfn, "w") ||
6096             die "unable to write config for VM $vmid\n";
6097
6098         my $cookie = { netcount => 0 };
6099         while (defined(my $line = <$fh>)) {
6100             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6101         }
6102
6103         $fh->close();
6104         $outfd->close();
6105     };
6106
6107     eval {
6108         # enable interrupts
6109         local $SIG{INT} =
6110             local $SIG{TERM} =
6111             local $SIG{QUIT} =
6112             local $SIG{HUP} =
6113             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6114         local $SIG{ALRM} = sub { die "got timeout\n"; };
6115
6116         $oldtimeout = alarm($timeout);
6117
6118         my $parser = sub {
6119             my $line = shift;
6120
6121             print "$line\n";
6122
6123             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6124                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6125                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6126             } elsif ($line =~ m/^CTIME: /) {
6127                 # we correctly received the vma config, so we can disable
6128                 # the timeout now for disk allocation (set to 10 minutes, so
6129                 # that we always timeout if something goes wrong)
6130                 alarm(600);
6131                 &$print_devmap();
6132                 print $fifofh "done\n";
6133                 my $tmp = $oldtimeout || 0;
6134                 $oldtimeout = undef;
6135                 alarm($tmp);
6136                 close($fifofh);
6137             }
6138         };
6139
6140         print "restore vma archive: $dbg_cmdstring\n";
6141         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6142     };
6143     my $err = $@;
6144
6145     alarm($oldtimeout) if $oldtimeout;
6146
6147     my $vollist = [];
6148     foreach my $devname (keys %$devinfo) {
6149         my $volid = $devinfo->{$devname}->{volid};
6150         push @$vollist, $volid if $volid;
6151     }
6152
6153     PVE::Storage::deactivate_volumes($cfg, $vollist);
6154
6155     unlink $mapfifo;
6156
6157     if ($err) {
6158         rmtree $tmpdir;
6159         unlink $tmpfn;
6160
6161         foreach my $devname (keys %$devinfo) {
6162             my $volid = $devinfo->{$devname}->{volid};
6163             next if !$volid;
6164             eval {
6165                 if ($volid =~ m|^/|) {
6166                     unlink $volid || die 'unlink failed\n';
6167                 } else {
6168                     PVE::Storage::vdisk_free($cfg, $volid);
6169                 }
6170                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6171             };
6172             print STDERR "unable to cleanup '$volid' - $@" if $@;
6173         }
6174         die $err;
6175     }
6176
6177     rmtree $tmpdir;
6178
6179     rename($tmpfn, $conffile) ||
6180         die "unable to commit configuration file '$conffile'\n";
6181
6182     PVE::Cluster::cfs_update(); # make sure we read new file
6183
6184     eval { rescan($vmid, 1); };
6185     warn $@ if $@;
6186 }
6187
6188 sub restore_tar_archive {
6189     my ($archive, $vmid, $user, $opts) = @_;
6190
6191     if ($archive ne '-') {
6192         my $firstfile = tar_archive_read_firstfile($archive);
6193         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6194             if $firstfile ne 'qemu-server.conf';
6195     }
6196
6197     my $storecfg = PVE::Storage::config();
6198
6199     # destroy existing data - keep empty config
6200     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6201     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6202
6203     my $tocmd = "/usr/lib/qemu-server/qmextract";
6204
6205     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6206     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6207     $tocmd .= ' --prealloc' if $opts->{prealloc};
6208     $tocmd .= ' --info' if $opts->{info};
6209
6210     # tar option "xf" does not autodetect compression when read from STDIN,
6211     # so we pipe to zcat
6212     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6213         PVE::Tools::shellquote("--to-command=$tocmd");
6214
6215     my $tmpdir = "/var/tmp/vzdumptmp$$";
6216     mkpath $tmpdir;
6217
6218     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6219     local $ENV{VZDUMP_VMID} = $vmid;
6220     local $ENV{VZDUMP_USER} = $user;
6221
6222     my $conffile = PVE::QemuConfig->config_file($vmid);
6223     my $tmpfn = "$conffile.$$.tmp";
6224
6225     # disable interrupts (always do cleanups)
6226     local $SIG{INT} =
6227         local $SIG{TERM} =
6228         local $SIG{QUIT} =
6229         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6230
6231     eval {
6232         # enable interrupts
6233         local $SIG{INT} =
6234             local $SIG{TERM} =
6235             local $SIG{QUIT} =
6236             local $SIG{HUP} =
6237             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6238
6239         if ($archive eq '-') {
6240             print "extracting archive from STDIN\n";
6241             run_command($cmd, input => "<&STDIN");
6242         } else {
6243             print "extracting archive '$archive'\n";
6244             run_command($cmd);
6245         }
6246
6247         return if $opts->{info};
6248
6249         # read new mapping
6250         my $map = {};
6251         my $statfile = "$tmpdir/qmrestore.stat";
6252         if (my $fd = IO::File->new($statfile, "r")) {
6253             while (defined (my $line = <$fd>)) {
6254                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6255                     $map->{$1} = $2 if $1;
6256                 } else {
6257                     print STDERR "unable to parse line in statfile - $line\n";
6258                 }
6259             }
6260             $fd->close();
6261         }
6262
6263         my $confsrc = "$tmpdir/qemu-server.conf";
6264
6265         my $srcfd = new IO::File($confsrc, "r") ||
6266             die "unable to open file '$confsrc'\n";
6267
6268         my $outfd = new IO::File ($tmpfn, "w") ||
6269             die "unable to write config for VM $vmid\n";
6270
6271         my $cookie = { netcount => 0 };
6272         while (defined (my $line = <$srcfd>)) {
6273             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6274         }
6275
6276         $srcfd->close();
6277         $outfd->close();
6278     };
6279     my $err = $@;
6280
6281     if ($err) {
6282
6283         unlink $tmpfn;
6284
6285         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6286
6287         die $err;
6288     }
6289
6290     rmtree $tmpdir;
6291
6292     rename $tmpfn, $conffile ||
6293         die "unable to commit configuration file '$conffile'\n";
6294
6295     PVE::Cluster::cfs_update(); # make sure we read new file
6296
6297     eval { rescan($vmid, 1); };
6298     warn $@ if $@;
6299 };
6300
6301 sub foreach_storage_used_by_vm {
6302     my ($conf, $func) = @_;
6303
6304     my $sidhash = {};
6305
6306     foreach_drive($conf, sub {
6307         my ($ds, $drive) = @_;
6308         return if drive_is_cdrom($drive);
6309
6310         my $volid = $drive->{file};
6311
6312         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6313         $sidhash->{$sid} = $sid if $sid;
6314     });
6315
6316     foreach my $sid (sort keys %$sidhash) {
6317         &$func($sid);
6318     }
6319 }
6320
6321 sub do_snapshots_with_qemu {
6322     my ($storecfg, $volid) = @_;
6323
6324     my $storage_name = PVE::Storage::parse_volume_id($volid);
6325
6326     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6327         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6328         return 1;
6329     }
6330
6331     if ($volid =~ m/\.(qcow2|qed)$/){
6332         return 1;
6333     }
6334
6335     return undef;
6336 }
6337
6338 sub qga_check_running {
6339     my ($vmid, $nowarn) = @_;
6340
6341     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6342     if ($@) {
6343         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6344         return 0;
6345     }
6346     return 1;
6347 }
6348
6349 sub template_create {
6350     my ($vmid, $conf, $disk) = @_;
6351
6352     my $storecfg = PVE::Storage::config();
6353
6354     foreach_drive($conf, sub {
6355         my ($ds, $drive) = @_;
6356
6357         return if drive_is_cdrom($drive);
6358         return if $disk && $ds ne $disk;
6359
6360         my $volid = $drive->{file};
6361         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6362
6363         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6364         $drive->{file} = $voliddst;
6365         $conf->{$ds} = print_drive($vmid, $drive);
6366         PVE::QemuConfig->write_config($vmid, $conf);
6367     });
6368 }
6369
6370 sub qemu_img_convert {
6371     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6372
6373     my $storecfg = PVE::Storage::config();
6374     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6375     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6376
6377     if ($src_storeid && $dst_storeid) {
6378
6379         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6380
6381         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6382         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6383
6384         my $src_format = qemu_img_format($src_scfg, $src_volname);
6385         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6386
6387         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6388         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6389
6390         my $cmd = [];
6391         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6392         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6393         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6394         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6395         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6396         if ($is_zero_initialized) {
6397             push @$cmd, "zeroinit:$dst_path";
6398         } else {
6399             push @$cmd, $dst_path;
6400         }
6401
6402         my $parser = sub {
6403             my $line = shift;
6404             if($line =~ m/\((\S+)\/100\%\)/){
6405                 my $percent = $1;
6406                 my $transferred = int($size * $percent / 100);
6407                 my $remaining = $size - $transferred;
6408
6409                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6410             }
6411
6412         };
6413
6414         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6415         my $err = $@;
6416         die "copy failed: $err" if $err;
6417     }
6418 }
6419
6420 sub qemu_img_format {
6421     my ($scfg, $volname) = @_;
6422
6423     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6424         return $1;
6425     } else {
6426         return "raw";
6427     }
6428 }
6429
6430 sub qemu_drive_mirror {
6431     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6432
6433     $jobs = {} if !$jobs;
6434
6435     my $qemu_target;
6436     my $format;
6437     $jobs->{"drive-$drive"} = {};
6438
6439     if ($dst_volid =~ /^nbd:/) {
6440         $qemu_target = $dst_volid;
6441         $format = "nbd";
6442     } else {
6443         my $storecfg = PVE::Storage::config();
6444         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6445
6446         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6447
6448         $format = qemu_img_format($dst_scfg, $dst_volname);
6449
6450         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6451
6452         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6453     }
6454
6455     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6456     $opts->{format} = $format if $format;
6457
6458     print "drive mirror is starting for drive-$drive\n";
6459
6460     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6461
6462     if (my $err = $@) {
6463         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6464         die "mirroring error: $err";
6465     }
6466
6467     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6468 }
6469
6470 sub qemu_drive_mirror_monitor {
6471     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6472
6473     eval {
6474         my $err_complete = 0;
6475
6476         while (1) {
6477             die "storage migration timed out\n" if $err_complete > 300;
6478
6479             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6480
6481             my $running_mirror_jobs = {};
6482             foreach my $stat (@$stats) {
6483                 next if $stat->{type} ne 'mirror';
6484                 $running_mirror_jobs->{$stat->{device}} = $stat;
6485             }
6486
6487             my $readycounter = 0;
6488
6489             foreach my $job (keys %$jobs) {
6490
6491                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6492                     print "$job : finished\n";
6493                     delete $jobs->{$job};
6494                     next;
6495                 }
6496
6497                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6498
6499                 my $busy = $running_mirror_jobs->{$job}->{busy};
6500                 my $ready = $running_mirror_jobs->{$job}->{ready};
6501                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6502                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6503                     my $remaining = $total - $transferred;
6504                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6505
6506                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6507                 }
6508
6509                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6510             }
6511
6512             last if scalar(keys %$jobs) == 0;
6513
6514             if ($readycounter == scalar(keys %$jobs)) {
6515                 print "all mirroring jobs are ready \n";
6516                 last if $skipcomplete; #do the complete later
6517
6518                 if ($vmiddst && $vmiddst != $vmid) {
6519                     my $agent_running = $qga && qga_check_running($vmid);
6520                     if ($agent_running) {
6521                         print "freeze filesystem\n";
6522                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6523                     } else {
6524                         print "suspend vm\n";
6525                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6526                     }
6527
6528                     # if we clone a disk for a new target vm, we don't switch the disk
6529                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6530
6531                     if ($agent_running) {
6532                         print "unfreeze filesystem\n";
6533                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6534                     } else {
6535                         print "resume vm\n";
6536                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6537                     }
6538
6539                     last;
6540                 } else {
6541
6542                     foreach my $job (keys %$jobs) {
6543                         # try to switch the disk if source and destination are on the same guest
6544                         print "$job: Completing block job...\n";
6545
6546                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6547                         if ($@ =~ m/cannot be completed/) {
6548                             print "$job: Block job cannot be completed, try again.\n";
6549                             $err_complete++;
6550                         }else {
6551                             print "$job: Completed successfully.\n";
6552                             $jobs->{$job}->{complete} = 1;
6553                         }
6554                     }
6555                 }
6556             }
6557             sleep 1;
6558         }
6559     };
6560     my $err = $@;
6561
6562     if ($err) {
6563         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6564         die "mirroring error: $err";
6565     }
6566
6567 }
6568
6569 sub qemu_blockjobs_cancel {
6570     my ($vmid, $jobs) = @_;
6571
6572     foreach my $job (keys %$jobs) {
6573         print "$job: Cancelling block job\n";
6574         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6575         $jobs->{$job}->{cancel} = 1;
6576     }
6577
6578     while (1) {
6579         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6580
6581         my $running_jobs = {};
6582         foreach my $stat (@$stats) {
6583             $running_jobs->{$stat->{device}} = $stat;
6584         }
6585
6586         foreach my $job (keys %$jobs) {
6587
6588             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6589                 print "$job: Done.\n";
6590                 delete $jobs->{$job};
6591             }
6592         }
6593
6594         last if scalar(keys %$jobs) == 0;
6595
6596         sleep 1;
6597     }
6598 }
6599
6600 sub clone_disk {
6601     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6602         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6603
6604     my $newvolid;
6605
6606     if (!$full) {
6607         print "create linked clone of drive $drivename ($drive->{file})\n";
6608         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6609         push @$newvollist, $newvolid;
6610     } else {
6611
6612         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6613         $storeid = $storage if $storage;
6614
6615         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6616         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6617
6618         print "create full clone of drive $drivename ($drive->{file})\n";
6619         my $name = undef;
6620         if (drive_is_cloudinit($drive)) {
6621             $name = "vm-$newvmid-cloudinit";
6622             # cloudinit only supports raw and qcow2 atm:
6623             if ($dst_format eq 'qcow2') {
6624                 $name .= '.qcow2';
6625             } elsif ($dst_format ne 'raw') {
6626                 die "clone: unhandled format for cloudinit image\n";
6627             }
6628         }
6629         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6630         push @$newvollist, $newvolid;
6631
6632         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6633
6634         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6635         if (!$running || $snapname) {
6636             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6637         } else {
6638
6639             my $kvmver = get_running_qemu_version ($vmid);
6640             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6641                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6642                     if $drive->{iothread};
6643             }
6644
6645             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6646         }
6647     }
6648
6649     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6650
6651     my $disk = $drive;
6652     $disk->{format} = undef;
6653     $disk->{file} = $newvolid;
6654     $disk->{size} = $size;
6655
6656     return $disk;
6657 }
6658
6659 # this only works if VM is running
6660 sub get_current_qemu_machine {
6661     my ($vmid) = @_;
6662
6663     my $cmd = { execute => 'query-machines', arguments => {} };
6664     my $res = vm_qmp_command($vmid, $cmd);
6665
6666     my ($current, $default);
6667     foreach my $e (@$res) {
6668         $default = $e->{name} if $e->{'is-default'};
6669         $current = $e->{name} if $e->{'is-current'};
6670     }
6671
6672     # fallback to the default machine if current is not supported by qemu
6673     return $current || $default || 'pc';
6674 }
6675
6676 sub get_running_qemu_version {
6677     my ($vmid) = @_;
6678     my $cmd = { execute => 'query-version', arguments => {} };
6679     my $res = vm_qmp_command($vmid, $cmd);
6680     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6681 }
6682
6683 sub qemu_machine_feature_enabled {
6684     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6685
6686     my $current_major;
6687     my $current_minor;
6688
6689     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6690
6691         $current_major = $3;
6692         $current_minor = $4;
6693
6694     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6695
6696         $current_major = $1;
6697         $current_minor = $2;
6698     }
6699
6700     return 1 if $current_major > $version_major ||
6701                 ($current_major == $version_major &&
6702                  $current_minor >= $version_minor);
6703 }
6704
6705 sub qemu_machine_pxe {
6706     my ($vmid, $conf, $machine) = @_;
6707
6708     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6709
6710     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6711         $machine .= '.pxe';
6712     }
6713
6714     return $machine;
6715 }
6716
6717 sub qemu_use_old_bios_files {
6718     my ($machine_type) = @_;
6719
6720     return if !$machine_type;
6721
6722     my $use_old_bios_files = undef;
6723
6724     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6725         $machine_type = $1;
6726         $use_old_bios_files = 1;
6727     } else {
6728         my $kvmver = kvm_user_version();
6729         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6730         # load new efi bios files on migration. So this hack is required to allow
6731         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6732         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6733         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6734     }
6735
6736     return ($use_old_bios_files, $machine_type);
6737 }
6738
6739 sub create_efidisk($$$$$) {
6740     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6741
6742     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6743     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6744
6745     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6746     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6747     PVE::Storage::activate_volumes($storecfg, [$volid]);
6748
6749     my $path = PVE::Storage::path($storecfg, $volid);
6750     eval {
6751         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6752     };
6753     die "Copying EFI vars image failed: $@" if $@;
6754
6755     return ($volid, $vars_size);
6756 }
6757
6758 sub vm_iothreads_list {
6759     my ($vmid) = @_;
6760
6761     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6762
6763     my $iothreads = {};
6764     foreach my $iothread (@$res) {
6765         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6766     }
6767
6768     return $iothreads;
6769 }
6770
6771 sub scsihw_infos {
6772     my ($conf, $drive) = @_;
6773
6774     my $maxdev = 0;
6775
6776     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6777         $maxdev = 7;
6778     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6779         $maxdev = 1;
6780     } else {
6781         $maxdev = 256;
6782     }
6783
6784     my $controller = int($drive->{index} / $maxdev);
6785     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6786
6787     return ($maxdev, $controller, $controller_prefix);
6788 }
6789
6790 sub add_hyperv_enlightenments {
6791     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6792
6793     return if $winversion < 6;
6794     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6795
6796     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6797
6798     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6799         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6800         push @$cpuFlags , 'hv_vapic';
6801         push @$cpuFlags , 'hv_time';
6802     } else {
6803         push @$cpuFlags , 'hv_spinlocks=0xffff';
6804     }
6805
6806     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6807         push @$cpuFlags , 'hv_reset';
6808         push @$cpuFlags , 'hv_vpindex';
6809         push @$cpuFlags , 'hv_runtime';
6810     }
6811
6812     if ($winversion >= 7) {
6813         push @$cpuFlags , 'hv_relaxed';
6814
6815         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6816             push @$cpuFlags , 'hv_synic';
6817             push @$cpuFlags , 'hv_stimer';
6818         }
6819     }
6820 }
6821
6822 sub windows_version {
6823     my ($ostype) = @_;
6824
6825     return 0 if !$ostype;
6826
6827     my $winversion = 0;
6828
6829     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6830         $winversion = 5;
6831     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6832         $winversion = 6;
6833     } elsif ($ostype =~ m/^win(\d+)$/) {
6834         $winversion = $1;
6835     }
6836
6837     return $winversion;
6838 }
6839
6840 sub resolve_dst_disk_format {
6841         my ($storecfg, $storeid, $src_volname, $format) = @_;
6842         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6843
6844         if (!$format) {
6845             # if no target format is specified, use the source disk format as hint
6846             if ($src_volname) {
6847                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6848                 $format = qemu_img_format($scfg, $src_volname);
6849             } else {
6850                 return $defFormat;
6851             }
6852         }
6853
6854         # test if requested format is supported - else use default
6855         my $supported = grep { $_ eq $format } @$validFormats;
6856         $format = $defFormat if !$supported;
6857         return $format;
6858 }
6859
6860 sub resolve_first_disk {
6861     my $conf = shift;
6862     my @disks = PVE::QemuServer::valid_drive_names();
6863     my $firstdisk;
6864     foreach my $ds (reverse @disks) {
6865         next if !$conf->{$ds};
6866         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6867         next if PVE::QemuServer::drive_is_cdrom($disk);
6868         $firstdisk = $ds;
6869     }
6870     return $firstdisk;
6871 }
6872
6873 sub generate_uuid {
6874     my ($uuid, $uuid_str);
6875     UUID::generate($uuid);
6876     UUID::unparse($uuid, $uuid_str);
6877     return $uuid_str;
6878 }
6879
6880 sub generate_smbios1_uuid {
6881     return "uuid=".generate_uuid();
6882 }
6883
6884 sub nbd_stop {
6885     my ($vmid) = @_;
6886
6887     vm_mon_cmd($vmid, 'nbd-server-stop');
6888 }
6889
6890 # bash completion helper
6891
6892 sub complete_backup_archives {
6893     my ($cmdname, $pname, $cvalue) = @_;
6894
6895     my $cfg = PVE::Storage::config();
6896
6897     my $storeid;
6898
6899     if ($cvalue =~ m/^([^:]+):/) {
6900         $storeid = $1;
6901     }
6902
6903     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6904
6905     my $res = [];
6906     foreach my $id (keys %$data) {
6907         foreach my $item (@{$data->{$id}}) {
6908             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6909             push @$res, $item->{volid} if defined($item->{volid});
6910         }
6911     }
6912
6913     return $res;
6914 }
6915
6916 my $complete_vmid_full = sub {
6917     my ($running) = @_;
6918
6919     my $idlist = vmstatus();
6920
6921     my $res = [];
6922
6923     foreach my $id (keys %$idlist) {
6924         my $d = $idlist->{$id};
6925         if (defined($running)) {
6926             next if $d->{template};
6927             next if $running && $d->{status} ne 'running';
6928             next if !$running && $d->{status} eq 'running';
6929         }
6930         push @$res, $id;
6931
6932     }
6933     return $res;
6934 };
6935
6936 sub complete_vmid {
6937     return &$complete_vmid_full();
6938 }
6939
6940 sub complete_vmid_stopped {
6941     return &$complete_vmid_full(0);
6942 }
6943
6944 sub complete_vmid_running {
6945     return &$complete_vmid_full(1);
6946 }
6947
6948 sub complete_storage {
6949
6950     my $cfg = PVE::Storage::config();
6951     my $ids = $cfg->{ids};
6952
6953     my $res = [];
6954     foreach my $sid (keys %$ids) {
6955         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6956         next if !$ids->{$sid}->{content}->{images};
6957         push @$res, $sid;
6958     }
6959
6960     return $res;
6961 }
6962
6963 1;