PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::SysFSTools;
  38 use PVE::Systemd;
  39 use Time::HiRes qw(gettimeofday);
  40 use File::Copy qw(copy);
  41 use URI::Escape;
  42
  43 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  44 my $OVMF = {
  45     x86_64 => [
  46         "$EDK2_FW_BASE/OVMF_CODE.fd",
  47         "$EDK2_FW_BASE/OVMF_VARS.fd"
  48     ],
  49     aarch64 => [
  50         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  51         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  52     ],
  53 };
  54
  55 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  56
  57 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  58
  59 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  60
  61 # Note about locking: we use flock on the config file protect
  62 # against concurent actions.
  63 # Aditionaly, we have a 'lock' setting in the config file. This
  64 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  65 # allowed when such lock is set. But you can ignore this kind of
  66 # lock with the --skiplock flag.
  67
  68 cfs_register_file('/qemu-server/',
  69                   \&parse_vm_config,
  70                   \&write_vm_config);
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  73     description => "Some command save/restore state from this location.",
  74     type => 'string',
  75     maxLength => 128,
  76     optional => 1,
  77 });
  78
  79 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  80     description => "The name of the snapshot.",
  81     type => 'string', format => 'pve-configid',
  82     maxLength => 40,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152
 153     # AMD CPUs
 154     athlon => 'AuthenticAMD',
 155     phenom  => 'AuthenticAMD',
 156     Opteron_G1  => 'AuthenticAMD',
 157     Opteron_G2  => 'AuthenticAMD',
 158     Opteron_G3  => 'AuthenticAMD',
 159     Opteron_G4  => 'AuthenticAMD',
 160     Opteron_G5  => 'AuthenticAMD',
 161     EPYC => 'AuthenticAMD',
 162     'EPYC-IBPB' => 'AuthenticAMD',
 163
 164     # generic types, use vendor from host node
 165     host => 'default',
 166     kvm32 => 'default',
 167     kvm64 => 'default',
 168     qemu32 => 'default',
 169     qemu64 => 'default',
 170     max => 'default',
 171 };
 172
 173 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 174
 175 my $cpu_fmt = {
 176     cputype => {
 177         description => "Emulated CPU type.",
 178         type => 'string',
 179         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 180         default => 'kvm64',
 181         default_key => 1,
 182     },
 183     hidden => {
 184         description => "Do not identify as a KVM virtual machine.",
 185         type => 'boolean',
 186         optional => 1,
 187         default => 0
 188     },
 189     flags => {
 190         description => "List of additional CPU flags separated by ';'."
 191                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 192                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 193         format_description => '+FLAG[;-FLAG...]',
 194         type => 'string',
 195         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 196         optional => 1,
 197     },
 198 };
 199
 200 my $watchdog_fmt = {
 201     model => {
 202         default_key => 1,
 203         type => 'string',
 204         enum => [qw(i6300esb ib700)],
 205         description => "Watchdog type to emulate.",
 206         default => 'i6300esb',
 207         optional => 1,
 208     },
 209     action => {
 210         type => 'string',
 211         enum => [qw(reset shutdown poweroff pause debug none)],
 212         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 213         optional => 1,
 214     },
 215 };
 216 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 217
 218 my $agent_fmt = {
 219     enabled => {
 220         description => "Enable/disable Qemu GuestAgent.",
 221         type => 'boolean',
 222         default => 0,
 223         default_key => 1,
 224     },
 225     fstrim_cloned_disks => {
 226         description => "Run fstrim after cloning/moving a disk.",
 227         type => 'boolean',
 228         optional => 1,
 229         default => 0
 230     },
 231 };
 232
 233 my $vga_fmt = {
 234     type => {
 235         description => "Select the VGA type.",
 236         type => 'string',
 237         default => 'std',
 238         optional => 1,
 239         default_key => 1,
 240         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 241     },
 242     memory => {
 243         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 244         type => 'integer',
 245         optional => 1,
 246         minimum => 4,
 247         maximum => 512,
 248     },
 249 };
 250
 251 my $confdesc = {
 252     onboot => {
 253         optional => 1,
 254         type => 'boolean',
 255         description => "Specifies whether a VM will be started during system bootup.",
 256         default => 0,
 257     },
 258     autostart => {
 259         optional => 1,
 260         type => 'boolean',
 261         description => "Automatic restart after crash (currently ignored).",
 262         default => 0,
 263     },
 264     hotplug => {
 265         optional => 1,
 266         type => 'string', format => 'pve-hotplug-features',
 267         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 268         default => 'network,disk,usb',
 269     },
 270     reboot => {
 271         optional => 1,
 272         type => 'boolean',
 273         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 274         default => 1,
 275     },
 276     lock => {
 277         optional => 1,
 278         type => 'string',
 279         description => "Lock/unlock the VM.",
 280         enum => [qw(migrate backup snapshot rollback)],
 281     },
 282     cpulimit => {
 283         optional => 1,
 284         type => 'number',
 285         description => "Limit of CPU usage.",
 286         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 287         minimum => 0,
 288         maximum => 128,
 289         default => 0,
 290     },
 291     cpuunits => {
 292         optional => 1,
 293         type => 'integer',
 294         description => "CPU weight for a VM.",
 295         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 296         minimum => 2,
 297         maximum => 262144,
 298         default => 1024,
 299     },
 300     memory => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 304         minimum => 16,
 305         default => 512,
 306     },
 307     balloon => {
 308         optional => 1,
 309         type => 'integer',
 310         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 311         minimum => 0,
 312     },
 313     shares => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 317         minimum => 0,
 318         maximum => 50000,
 319         default => 1000,
 320     },
 321     keyboard => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 325                        "It should not be necessary to set it.",
 326         enum => PVE::Tools::kvmkeymaplist(),
 327         default => undef,
 328     },
 329     name => {
 330         optional => 1,
 331         type => 'string', format => 'dns-name',
 332         description => "Set a name for the VM. Only used on the configuration web interface.",
 333     },
 334     scsihw => {
 335         optional => 1,
 336         type => 'string',
 337         description => "SCSI controller model",
 338         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 339         default => 'lsi',
 340     },
 341     description => {
 342         optional => 1,
 343         type => 'string',
 344         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 345     },
 346     ostype => {
 347         optional => 1,
 348         type => 'string',
 349         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 350         description => "Specify guest operating system.",
 351         verbose_description => <<EODESC,
 352 Specify guest operating system. This is used to enable special
 353 optimization/features for specific operating systems:
 354
 355 [horizontal]
 356 other;; unspecified OS
 357 wxp;; Microsoft Windows XP
 358 w2k;; Microsoft Windows 2000
 359 w2k3;; Microsoft Windows 2003
 360 w2k8;; Microsoft Windows 2008
 361 wvista;; Microsoft Windows Vista
 362 win7;; Microsoft Windows 7
 363 win8;; Microsoft Windows 8/2012/2012r2
 364 win10;; Microsoft Windows 10/2016
 365 l24;; Linux 2.4 Kernel
 366 l26;; Linux 2.6/3.X Kernel
 367 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 368 EODESC
 369     },
 370     boot => {
 371         optional => 1,
 372         type => 'string',
 373         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 374         pattern => '[acdn]{1,4}',
 375         default => 'cdn',
 376     },
 377     bootdisk => {
 378         optional => 1,
 379         type => 'string', format => 'pve-qm-bootdisk',
 380         description => "Enable booting from specified disk.",
 381         pattern => '(ide|sata|scsi|virtio)\d+',
 382     },
 383     smp => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPUs. Please use option -sockets instead.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     sockets => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of CPU sockets.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     cores => {
 398         optional => 1,
 399         type => 'integer',
 400         description => "The number of cores per socket.",
 401         minimum => 1,
 402         default => 1,
 403     },
 404     numa => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable NUMA.",
 408         default => 0,
 409     },
 410     hugepages => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Enable/disable hugepages memory.",
 414         enum => [qw(any 2 1024)],
 415     },
 416     vcpus => {
 417         optional => 1,
 418         type => 'integer',
 419         description => "Number of hotplugged vcpus.",
 420         minimum => 1,
 421         default => 0,
 422     },
 423     acpi => {
 424         optional => 1,
 425         type => 'boolean',
 426         description => "Enable/disable ACPI.",
 427         default => 1,
 428     },
 429     agent => {
 430         optional => 1,
 431         description => "Enable/disable Qemu GuestAgent and its properties.",
 432         type => 'string',
 433         format => $agent_fmt,
 434     },
 435     kvm => {
 436         optional => 1,
 437         type => 'boolean',
 438         description => "Enable/disable KVM hardware virtualization.",
 439         default => 1,
 440     },
 441     tdf => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable time drift fix.",
 445         default => 0,
 446     },
 447     localtime => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 451     },
 452     freeze => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 456     },
 457     vga => {
 458         optional => 1,
 459         type => 'string', format => $vga_fmt,
 460         description => "Configure the VGA hardware.",
 461         verbose_description => "Configure the VGA Hardware. If you want to use ".
 462             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 463             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 464             "is 'std' for all OS types besides some Windows versions (XP and " .
 465             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 466             "display server. For win* OS you can select how many independent " .
 467             "displays you want, Linux guests can add displays them self.\n".
 468             "You can also run without any graphic card, using a serial device as terminal.",
 469     },
 470     watchdog => {
 471         optional => 1,
 472         type => 'string', format => 'pve-qm-watchdog',
 473         description => "Create a virtual hardware watchdog device.",
 474         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 475             " (by a guest action), the watchdog must be periodically polled " .
 476             "by an agent inside the guest or else the watchdog will reset " .
 477             "the guest (or execute the respective action specified)",
 478     },
 479     startdate => {
 480         optional => 1,
 481         type => 'string',
 482         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 483         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 484         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 485         default => 'now',
 486     },
 487     startup =>  get_standard_option('pve-startup-order'),
 488     template => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable Template.",
 492         default => 0,
 493     },
 494     args => {
 495         optional => 1,
 496         type => 'string',
 497         description => "Arbitrary arguments passed to kvm.",
 498         verbose_description => <<EODESCR,
 499 Arbitrary arguments passed to kvm, for example:
 500
 501 args: -no-reboot -no-hpet
 502
 503 NOTE: this option is for experts only.
 504 EODESCR
 505     },
 506     tablet => {
 507         optional => 1,
 508         type => 'boolean',
 509         default => 1,
 510         description => "Enable/disable the USB tablet device.",
 511         verbose_description => "Enable/disable the USB tablet device. This device is " .
 512             "usually needed to allow absolute mouse positioning with VNC. " .
 513             "Else the mouse runs out of sync with normal VNC clients. " .
 514             "If you're running lots of console-only guests on one host, " .
 515             "you may consider disabling this to save some context switches. " .
 516             "This is turned off by default if you use spice (-vga=qxl).",
 517     },
 518     migrate_speed => {
 519         optional => 1,
 520         type => 'integer',
 521         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 522         minimum => 0,
 523         default => 0,
 524     },
 525     migrate_downtime => {
 526         optional => 1,
 527         type => 'number',
 528         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 529         minimum => 0,
 530         default => 0.1,
 531     },
 532     cdrom => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-ide',
 535         typetext => '<volume>',
 536         description => "This is an alias for option -ide2",
 537     },
 538     cpu => {
 539         optional => 1,
 540         description => "Emulated CPU type.",
 541         type => 'string',
 542         format => $cpu_fmt,
 543     },
 544     parent => get_standard_option('pve-snapshot-name', {
 545         optional => 1,
 546         description => "Parent snapshot name. This is used internally, and should not be modified.",
 547     }),
 548     snaptime => {
 549         optional => 1,
 550         description => "Timestamp for snapshots.",
 551         type => 'integer',
 552         minimum => 0,
 553     },
 554     vmstate => {
 555         optional => 1,
 556         type => 'string', format => 'pve-volume-id',
 557         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 558     },
 559     vmstatestorage => get_standard_option('pve-storage-id', {
 560         description => "Default storage for VM state volumes/files.",
 561         optional => 1,
 562     }),
 563     runningmachine => get_standard_option('pve-qemu-machine', {
 564         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 565     }),
 566     machine => get_standard_option('pve-qemu-machine'),
 567     arch => {
 568         description => "Virtual processor architecture. Defaults to the host.",
 569         optional => 1,
 570         type => 'string',
 571         enum => [qw(x86_64 aarch64)],
 572     },
 573     smbios1 => {
 574         description => "Specify SMBIOS type 1 fields.",
 575         type => 'string', format => 'pve-qm-smbios1',
 576         maxLength => 256,
 577         optional => 1,
 578     },
 579     protection => {
 580         optional => 1,
 581         type => 'boolean',
 582         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 583         default => 0,
 584     },
 585     bios => {
 586         optional => 1,
 587         type => 'string',
 588         enum => [ qw(seabios ovmf) ],
 589         description => "Select BIOS implementation.",
 590         default => 'seabios',
 591     },
 592     vmgenid => {
 593         type => 'string',
 594         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 595         format_description => 'UUID',
 596         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 597         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 598             " 128-bit integer value identifier to the guest OS. This allows to".
 599             " notify the guest operating system when the virtual machine is".
 600             " executed with a different configuration (e.g. snapshot execution".
 601             " or creation from a template). The guest operating system notices".
 602             " the change, and is then able to react as appropriate by marking".
 603             " its copies of distributed databases as dirty, re-initializing its".
 604             " random number generator, etc.\n".
 605             "Note that auto-creation only works when done throug API/CLI create".
 606             " or update methods, but not when manually editing the config file.",
 607         default => "1 (autogenerated)",
 608         optional => 1,
 609     },
 610 };
 611
 612 my $confdesc_cloudinit = {
 613     citype => {
 614         optional => 1,
 615         type => 'string',
 616         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 617         enum => ['configdrive2', 'nocloud'],
 618     },
 619     ciuser => {
 620         optional => 1,
 621         type => 'string',
 622         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 623     },
 624     cipassword => {
 625         optional => 1,
 626         type => 'string',
 627         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 628     },
 629     searchdomain => {
 630         optional => 1,
 631         type => 'string',
 632         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 633     },
 634     nameserver => {
 635         optional => 1,
 636         type => 'string', format => 'address-list',
 637         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 638     },
 639     sshkeys => {
 640         optional => 1,
 641         type => 'string',
 642         format => 'urlencoded',
 643         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 644     },
 645 };
 646
 647 # what about other qemu settings ?
 648 #cpu => 'string',
 649 #machine => 'string',
 650 #fda => 'file',
 651 #fdb => 'file',
 652 #mtdblock => 'file',
 653 #sd => 'file',
 654 #pflash => 'file',
 655 #snapshot => 'bool',
 656 #bootp => 'file',
 657 ##tftp => 'dir',
 658 ##smb => 'dir',
 659 #kernel => 'file',
 660 #append => 'string',
 661 #initrd => 'file',
 662 ##soundhw => 'string',
 663
 664 while (my ($k, $v) = each %$confdesc) {
 665     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 666 }
 667
 668 my $MAX_IDE_DISKS = 4;
 669 my $MAX_SCSI_DISKS = 14;
 670 my $MAX_VIRTIO_DISKS = 16;
 671 my $MAX_SATA_DISKS = 6;
 672 my $MAX_USB_DEVICES = 5;
 673 my $MAX_NETS = 32;
 674 my $MAX_UNUSED_DISKS = 256;
 675 my $MAX_HOSTPCI_DEVICES = 4;
 676 my $MAX_SERIAL_PORTS = 4;
 677 my $MAX_PARALLEL_PORTS = 3;
 678 my $MAX_NUMA = 8;
 679
 680 my $numa_fmt = {
 681     cpus => {
 682         type => "string",
 683         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 684         description => "CPUs accessing this NUMA node.",
 685         format_description => "id[-id];...",
 686     },
 687     memory => {
 688         type => "number",
 689         description => "Amount of memory this NUMA node provides.",
 690         optional => 1,
 691     },
 692     hostnodes => {
 693         type => "string",
 694         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 695         description => "Host NUMA nodes to use.",
 696         format_description => "id[-id];...",
 697         optional => 1,
 698     },
 699     policy => {
 700         type => 'string',
 701         enum => [qw(preferred bind interleave)],
 702         description => "NUMA allocation policy.",
 703         optional => 1,
 704     },
 705 };
 706 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 707 my $numadesc = {
 708     optional => 1,
 709     type => 'string', format => $numa_fmt,
 710     description => "NUMA topology.",
 711 };
 712 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 713
 714 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 715     $confdesc->{"numa$i"} = $numadesc;
 716 }
 717
 718 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 719                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 720                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 721 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 722
 723 my $net_fmt_bridge_descr = <<__EOD__;
 724 Bridge to attach the network device to. The Proxmox VE standard bridge
 725 is called 'vmbr0'.
 726
 727 If you do not specify a bridge, we create a kvm user (NATed) network
 728 device, which provides DHCP and DNS services. The following addresses
 729 are used:
 730
 731  10.0.2.2   Gateway
 732  10.0.2.3   DNS Server
 733  10.0.2.4   SMB Server
 734
 735 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 736 __EOD__
 737
 738 my $net_fmt = {
 739     macaddr => {
 740         type => 'string',
 741         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 742         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 743         format_description => "XX:XX:XX:XX:XX:XX",
 744         optional => 1,
 745     },
 746     model => {
 747         type => 'string',
 748         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 749         enum => $nic_model_list,
 750         default_key => 1,
 751     },
 752     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 753     bridge => {
 754         type => 'string',
 755         description => $net_fmt_bridge_descr,
 756         format_description => 'bridge',
 757         optional => 1,
 758     },
 759     queues => {
 760         type => 'integer',
 761         minimum => 0, maximum => 16,
 762         description => 'Number of packet queues to be used on the device.',
 763         optional => 1,
 764     },
 765     rate => {
 766         type => 'number',
 767         minimum => 0,
 768         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 769         optional => 1,
 770     },
 771     tag => {
 772         type => 'integer',
 773         minimum => 1, maximum => 4094,
 774         description => 'VLAN tag to apply to packets on this interface.',
 775         optional => 1,
 776     },
 777     trunks => {
 778         type => 'string',
 779         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 780         description => 'VLAN trunks to pass through this interface.',
 781         format_description => 'vlanid[;vlanid...]',
 782         optional => 1,
 783     },
 784     firewall => {
 785         type => 'boolean',
 786         description => 'Whether this interface should be protected by the firewall.',
 787         optional => 1,
 788     },
 789     link_down => {
 790         type => 'boolean',
 791         description => 'Whether this interface should be disconnected (like pulling the plug).',
 792         optional => 1,
 793     },
 794 };
 795
 796 my $netdesc = {
 797     optional => 1,
 798     type => 'string', format => $net_fmt,
 799     description => "Specify network devices.",
 800 };
 801
 802 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 803
 804 my $ipconfig_fmt = {
 805     ip => {
 806         type => 'string',
 807         format => 'pve-ipv4-config',
 808         format_description => 'IPv4Format/CIDR',
 809         description => 'IPv4 address in CIDR format.',
 810         optional => 1,
 811         default => 'dhcp',
 812     },
 813     gw => {
 814         type => 'string',
 815         format => 'ipv4',
 816         format_description => 'GatewayIPv4',
 817         description => 'Default gateway for IPv4 traffic.',
 818         optional => 1,
 819         requires => 'ip',
 820     },
 821     ip6 => {
 822         type => 'string',
 823         format => 'pve-ipv6-config',
 824         format_description => 'IPv6Format/CIDR',
 825         description => 'IPv6 address in CIDR format.',
 826         optional => 1,
 827         default => 'dhcp',
 828     },
 829     gw6 => {
 830         type => 'string',
 831         format => 'ipv6',
 832         format_description => 'GatewayIPv6',
 833         description => 'Default gateway for IPv6 traffic.',
 834         optional => 1,
 835         requires => 'ip6',
 836     },
 837 };
 838 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 839 my $ipconfigdesc = {
 840     optional => 1,
 841     type => 'string', format => 'pve-qm-ipconfig',
 842     description => <<'EODESCR',
 843 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 844
 845 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 846
 847 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 848 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 849
 850 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 851 EODESCR
 852 };
 853 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 854
 855 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 856     $confdesc->{"net$i"} = $netdesc;
 857     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 858 }
 859
 860 foreach my $key (keys %$confdesc_cloudinit) {
 861     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 862 }
 863
 864 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 865 sub verify_volume_id_or_qm_path {
 866     my ($volid, $noerr) = @_;
 867
 868     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 869         return $volid;
 870     }
 871
 872     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 873     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 874     if ($@) {
 875         return undef if $noerr;
 876         die $@;
 877     }
 878     return $volid;
 879 }
 880
 881 my $drivename_hash;
 882
 883 my %drivedesc_base = (
 884     volume => { alias => 'file' },
 885     file => {
 886         type => 'string',
 887         format => 'pve-volume-id-or-qm-path',
 888         default_key => 1,
 889         format_description => 'volume',
 890         description => "The drive's backing volume.",
 891     },
 892     media => {
 893         type => 'string',
 894         enum => [qw(cdrom disk)],
 895         description => "The drive's media type.",
 896         default => 'disk',
 897         optional => 1
 898     },
 899     cyls => {
 900         type => 'integer',
 901         description => "Force the drive's physical geometry to have a specific cylinder count.",
 902         optional => 1
 903     },
 904     heads => {
 905         type => 'integer',
 906         description => "Force the drive's physical geometry to have a specific head count.",
 907         optional => 1
 908     },
 909     secs => {
 910         type => 'integer',
 911         description => "Force the drive's physical geometry to have a specific sector count.",
 912         optional => 1
 913     },
 914     trans => {
 915         type => 'string',
 916         enum => [qw(none lba auto)],
 917         description => "Force disk geometry bios translation mode.",
 918         optional => 1,
 919     },
 920     snapshot => {
 921         type => 'boolean',
 922         description => "Controls qemu's snapshot mode feature."
 923             . " If activated, changes made to the disk are temporary and will"
 924             . " be discarded when the VM is shutdown.",
 925         optional => 1,
 926     },
 927     cache => {
 928         type => 'string',
 929         enum => [qw(none writethrough writeback unsafe directsync)],
 930         description => "The drive's cache mode",
 931         optional => 1,
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941     backup => {
 942         type => 'boolean',
 943         description => "Whether the drive should be included when making backups.",
 944         optional => 1,
 945     },
 946     replicate => {
 947         type => 'boolean',
 948         description => 'Whether the drive should considered for replication jobs.',
 949         optional => 1,
 950         default => 1,
 951     },
 952     rerror => {
 953         type => 'string',
 954         enum => [qw(ignore report stop)],
 955         description => 'Read error action.',
 956         optional => 1,
 957     },
 958     werror => {
 959         type => 'string',
 960         enum => [qw(enospc ignore report stop)],
 961         description => 'Write error action.',
 962         optional => 1,
 963     },
 964     aio => {
 965         type => 'string',
 966         enum => [qw(native threads)],
 967         description => 'AIO type to use.',
 968         optional => 1,
 969     },
 970     discard => {
 971         type => 'string',
 972         enum => [qw(ignore on)],
 973         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 974         optional => 1,
 975     },
 976     detect_zeroes => {
 977         type => 'boolean',
 978         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 979         optional => 1,
 980     },
 981     serial => {
 982         type => 'string',
 983         format => 'urlencoded',
 984         format_description => 'serial',
 985         maxLength => 20*3, # *3 since it's %xx url enoded
 986         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 987         optional => 1,
 988     },
 989     shared => {
 990         type => 'boolean',
 991         description => 'Mark this locally-managed volume as available on all nodes',
 992         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 993         optional => 1,
 994         default => 0,
 995     }
 996 );
 997
 998 my %iothread_fmt = ( iothread => {
 999         type => 'boolean',
1000         description => "Whether to use iothreads for this drive",
1001         optional => 1,
1002 });
1003
1004 my %model_fmt = (
1005     model => {
1006         type => 'string',
1007         format => 'urlencoded',
1008         format_description => 'model',
1009         maxLength => 40*3, # *3 since it's %xx url enoded
1010         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1011         optional => 1,
1012     },
1013 );
1014
1015 my %queues_fmt = (
1016     queues => {
1017         type => 'integer',
1018         description => "Number of queues.",
1019         minimum => 2,
1020         optional => 1
1021     }
1022 );
1023
1024 my %scsiblock_fmt = (
1025     scsiblock => {
1026         type => 'boolean',
1027         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1028         optional => 1,
1029         default => 0,
1030     },
1031 );
1032
1033 my %ssd_fmt = (
1034     ssd => {
1035         type => 'boolean',
1036         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1037         optional => 1,
1038     },
1039 );
1040
1041 my $add_throttle_desc = sub {
1042     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1043     my $d = {
1044         type => $type,
1045         format_description => $unit,
1046         description => "Maximum $what in $longunit.",
1047         optional => 1,
1048     };
1049     $d->{minimum} = $minimum if defined($minimum);
1050     $drivedesc_base{$key} = $d;
1051 };
1052 # throughput: (leaky bucket)
1053 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1054 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1056 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1057 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1059 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1060 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1061 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1062
1063 # pools: (pool of IO before throttling starts taking effect)
1064 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1065 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1067 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1068 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1069 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1070
1071 # burst lengths
1072 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1073 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1075 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1078
1079 # legacy support
1080 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1081 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1082 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1083 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1084
1085 my $ide_fmt = {
1086     %drivedesc_base,
1087     %model_fmt,
1088     %ssd_fmt,
1089 };
1090 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1091
1092 my $idedesc = {
1093     optional => 1,
1094     type => 'string', format => $ide_fmt,
1095     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1096 };
1097 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1098
1099 my $scsi_fmt = {
1100     %drivedesc_base,
1101     %iothread_fmt,
1102     %queues_fmt,
1103     %scsiblock_fmt,
1104     %ssd_fmt,
1105 };
1106 my $scsidesc = {
1107     optional => 1,
1108     type => 'string', format => $scsi_fmt,
1109     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1112
1113 my $sata_fmt = {
1114     %drivedesc_base,
1115     %ssd_fmt,
1116 };
1117 my $satadesc = {
1118     optional => 1,
1119     type => 'string', format => $sata_fmt,
1120     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1123
1124 my $virtio_fmt = {
1125     %drivedesc_base,
1126     %iothread_fmt,
1127 };
1128 my $virtiodesc = {
1129     optional => 1,
1130     type => 'string', format => $virtio_fmt,
1131     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1132 };
1133 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1134
1135 my $alldrive_fmt = {
1136     %drivedesc_base,
1137     %iothread_fmt,
1138     %model_fmt,
1139     %queues_fmt,
1140     %scsiblock_fmt,
1141     %ssd_fmt,
1142 };
1143
1144 my $efidisk_fmt = {
1145     volume => { alias => 'file' },
1146     file => {
1147         type => 'string',
1148         format => 'pve-volume-id-or-qm-path',
1149         default_key => 1,
1150         format_description => 'volume',
1151         description => "The drive's backing volume.",
1152     },
1153     format => get_standard_option('pve-qm-image-format'),
1154     size => {
1155         type => 'string',
1156         format => 'disk-size',
1157         format_description => 'DiskSize',
1158         description => "Disk size. This is purely informational and has no effect.",
1159         optional => 1,
1160     },
1161 };
1162
1163 my $efidisk_desc = {
1164     optional => 1,
1165     type => 'string', format => $efidisk_fmt,
1166     description => "Configure a Disk for storing EFI vars",
1167 };
1168
1169 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1170
1171 my $usb_fmt = {
1172     host => {
1173         default_key => 1,
1174         type => 'string', format => 'pve-qm-usb-device',
1175         format_description => 'HOSTUSBDEVICE|spice',
1176         description => <<EODESCR,
1177 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1178
1179  'bus-port(.port)*' (decimal numbers) or
1180  'vendor_id:product_id' (hexadeciaml numbers) or
1181  'spice'
1182
1183 You can use the 'lsusb -t' command to list existing usb devices.
1184
1185 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1186
1187 The value 'spice' can be used to add a usb redirection devices for spice.
1188 EODESCR
1189     },
1190     usb3 => {
1191         optional => 1,
1192         type => 'boolean',
1193         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1194         default => 0,
1195     },
1196 };
1197
1198 my $usbdesc = {
1199     optional => 1,
1200     type => 'string', format => $usb_fmt,
1201     description => "Configure an USB device (n is 0 to 4).",
1202 };
1203 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1204
1205 # NOTE: the match-groups of this regex are used in parse_hostpci
1206 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1207 my $hostpci_fmt = {
1208     host => {
1209         default_key => 1,
1210         type => 'string',
1211         pattern => qr/$PCIRE(;$PCIRE)*/,
1212         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1213         description => <<EODESCR,
1214 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1215 of PCI virtual functions of the host. HOSTPCIID syntax is:
1216
1217 'bus:dev.func' (hexadecimal numbers)
1218
1219 You can us the 'lspci' command to list existing PCI devices.
1220 EODESCR
1221     },
1222     rombar => {
1223         type => 'boolean',
1224         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1225         optional => 1,
1226         default => 1,
1227     },
1228     romfile => {
1229         type => 'string',
1230         pattern => '[^,;]+',
1231         format_description => 'string',
1232         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1233         optional => 1,
1234     },
1235     pcie => {
1236         type => 'boolean',
1237         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1238         optional => 1,
1239         default => 0,
1240     },
1241     'x-vga' => {
1242         type => 'boolean',
1243         description =>  "Enable vfio-vga device support.",
1244         optional => 1,
1245         default => 0,
1246     },
1247 };
1248 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1249
1250 my $hostpcidesc = {
1251         optional => 1,
1252         type => 'string', format => 'pve-qm-hostpci',
1253         description => "Map host PCI devices into guest.",
1254         verbose_description =>  <<EODESCR,
1255 Map host PCI devices into guest.
1256
1257 NOTE: This option allows direct access to host hardware. So it is no longer
1258 possible to migrate such machines - use with special care.
1259
1260 CAUTION: Experimental! User reported problems with this option.
1261 EODESCR
1262 };
1263 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1264
1265 my $serialdesc = {
1266         optional => 1,
1267         type => 'string',
1268         pattern => '(/dev/.+|socket)',
1269         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1270         verbose_description =>  <<EODESCR,
1271 Create a serial device inside the VM (n is 0 to 3), and pass through a
1272 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1273 host side (use 'qm terminal' to open a terminal connection).
1274
1275 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1276
1277 CAUTION: Experimental! User reported problems with this option.
1278 EODESCR
1279 };
1280
1281 my $paralleldesc= {
1282         optional => 1,
1283         type => 'string',
1284         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1285         description =>  "Map host parallel devices (n is 0 to 2).",
1286         verbose_description =>  <<EODESCR,
1287 Map host parallel devices (n is 0 to 2).
1288
1289 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1290
1291 CAUTION: Experimental! User reported problems with this option.
1292 EODESCR
1293 };
1294
1295 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1296     $confdesc->{"parallel$i"} = $paralleldesc;
1297 }
1298
1299 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1300     $confdesc->{"serial$i"} = $serialdesc;
1301 }
1302
1303 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1304     $confdesc->{"hostpci$i"} = $hostpcidesc;
1305 }
1306
1307 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1308     $drivename_hash->{"ide$i"} = 1;
1309     $confdesc->{"ide$i"} = $idedesc;
1310 }
1311
1312 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1313     $drivename_hash->{"sata$i"} = 1;
1314     $confdesc->{"sata$i"} = $satadesc;
1315 }
1316
1317 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1318     $drivename_hash->{"scsi$i"} = 1;
1319     $confdesc->{"scsi$i"} = $scsidesc ;
1320 }
1321
1322 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1323     $drivename_hash->{"virtio$i"} = 1;
1324     $confdesc->{"virtio$i"} = $virtiodesc;
1325 }
1326
1327 $drivename_hash->{efidisk0} = 1;
1328 $confdesc->{efidisk0} = $efidisk_desc;
1329
1330 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1331     $confdesc->{"usb$i"} = $usbdesc;
1332 }
1333
1334 my $unuseddesc = {
1335     optional => 1,
1336     type => 'string', format => 'pve-volume-id',
1337     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1338 };
1339
1340 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1341     $confdesc->{"unused$i"} = $unuseddesc;
1342 }
1343
1344 my $kvm_api_version = 0;
1345
1346 sub kvm_version {
1347     return $kvm_api_version if $kvm_api_version;
1348
1349     open my $fh, '<', '/dev/kvm'
1350         or return undef;
1351
1352     # 0xae00 => KVM_GET_API_VERSION
1353     $kvm_api_version = ioctl($fh, 0xae00, 0);
1354
1355     return $kvm_api_version;
1356 }
1357
1358 my $kvm_user_version;
1359
1360 sub kvm_user_version {
1361
1362     return $kvm_user_version if $kvm_user_version;
1363
1364     $kvm_user_version = 'unknown';
1365
1366     my $code = sub {
1367         my $line = shift;
1368         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1369             $kvm_user_version = $2;
1370         }
1371     };
1372
1373     eval { run_command("kvm -version", outfunc => $code); };
1374     warn $@ if $@;
1375
1376     return $kvm_user_version;
1377
1378 }
1379
1380 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1381
1382 sub valid_drive_names {
1383     # order is important - used to autoselect boot disk
1384     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1385             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1386             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1387             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1388             'efidisk0');
1389 }
1390
1391 sub is_valid_drivename {
1392     my $dev = shift;
1393
1394     return defined($drivename_hash->{$dev});
1395 }
1396
1397 sub option_exists {
1398     my $key = shift;
1399     return defined($confdesc->{$key});
1400 }
1401
1402 sub nic_models {
1403     return $nic_model_list;
1404 }
1405
1406 sub os_list_description {
1407
1408     return {
1409         other => 'Other',
1410         wxp => 'Windows XP',
1411         w2k => 'Windows 2000',
1412         w2k3 =>, 'Windows 2003',
1413         w2k8 => 'Windows 2008',
1414         wvista => 'Windows Vista',
1415         win7 => 'Windows 7',
1416         win8 => 'Windows 8/2012',
1417         win10 => 'Windows 10/2016',
1418         l24 => 'Linux 2.4',
1419         l26 => 'Linux 2.6',
1420     };
1421 }
1422
1423 my $cdrom_path;
1424
1425 sub get_cdrom_path {
1426
1427     return  $cdrom_path if $cdrom_path;
1428
1429     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1430     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1431     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1432 }
1433
1434 sub get_iso_path {
1435     my ($storecfg, $vmid, $cdrom) = @_;
1436
1437     if ($cdrom eq 'cdrom') {
1438         return get_cdrom_path();
1439     } elsif ($cdrom eq 'none') {
1440         return '';
1441     } elsif ($cdrom =~ m|^/|) {
1442         return $cdrom;
1443     } else {
1444         return PVE::Storage::path($storecfg, $cdrom);
1445     }
1446 }
1447
1448 # try to convert old style file names to volume IDs
1449 sub filename_to_volume_id {
1450     my ($vmid, $file, $media) = @_;
1451
1452      if (!($file eq 'none' || $file eq 'cdrom' ||
1453           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1454
1455         return undef if $file =~ m|/|;
1456
1457         if ($media && $media eq 'cdrom') {
1458             $file = "local:iso/$file";
1459         } else {
1460             $file = "local:$vmid/$file";
1461         }
1462     }
1463
1464     return $file;
1465 }
1466
1467 sub verify_media_type {
1468     my ($opt, $vtype, $media) = @_;
1469
1470     return if !$media;
1471
1472     my $etype;
1473     if ($media eq 'disk') {
1474         $etype = 'images';
1475     } elsif ($media eq 'cdrom') {
1476         $etype = 'iso';
1477     } else {
1478         die "internal error";
1479     }
1480
1481     return if ($vtype eq $etype);
1482
1483     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1484 }
1485
1486 sub cleanup_drive_path {
1487     my ($opt, $storecfg, $drive) = @_;
1488
1489     # try to convert filesystem paths to volume IDs
1490
1491     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1492         ($drive->{file} !~ m|^/dev/.+|) &&
1493         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1494         ($drive->{file} !~ m/^\d+$/)) {
1495         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1496         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1497         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1498         verify_media_type($opt, $vtype, $drive->{media});
1499         $drive->{file} = $volid;
1500     }
1501
1502     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1503 }
1504
1505 sub parse_hotplug_features {
1506     my ($data) = @_;
1507
1508     my $res = {};
1509
1510     return $res if $data eq '0';
1511
1512     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1513
1514     foreach my $feature (PVE::Tools::split_list($data)) {
1515         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1516             $res->{$1} = 1;
1517         } else {
1518             die "invalid hotplug feature '$feature'\n";
1519         }
1520     }
1521     return $res;
1522 }
1523
1524 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1525 sub pve_verify_hotplug_features {
1526     my ($value, $noerr) = @_;
1527
1528     return $value if parse_hotplug_features($value);
1529
1530     return undef if $noerr;
1531
1532     die "unable to parse hotplug option\n";
1533 }
1534
1535 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1536 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1537 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1538 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1539 #        [,iothread=on][,serial=serial][,model=model]
1540
1541 sub parse_drive {
1542     my ($key, $data) = @_;
1543
1544     my ($interface, $index);
1545
1546     if ($key =~ m/^([^\d]+)(\d+)$/) {
1547         $interface = $1;
1548         $index = $2;
1549     } else {
1550         return undef;
1551     }
1552
1553     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1554                                      : $confdesc->{$key}->{format};
1555     if (!$desc) {
1556         warn "invalid drive key: $key\n";
1557         return undef;
1558     }
1559     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1560     return undef if !$res;
1561     $res->{interface} = $interface;
1562     $res->{index} = $index;
1563
1564     my $error = 0;
1565     foreach my $opt (qw(bps bps_rd bps_wr)) {
1566         if (my $bps = defined(delete $res->{$opt})) {
1567             if (defined($res->{"m$opt"})) {
1568                 warn "both $opt and m$opt specified\n";
1569                 ++$error;
1570                 next;
1571             }
1572             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1573         }
1574     }
1575
1576     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1577     for my $requirement (
1578         [mbps_max => 'mbps'],
1579         [mbps_rd_max => 'mbps_rd'],
1580         [mbps_wr_max => 'mbps_wr'],
1581         [miops_max => 'miops'],
1582         [miops_rd_max => 'miops_rd'],
1583         [miops_wr_max => 'miops_wr'],
1584         [bps_max_length => 'mbps_max'],
1585         [bps_rd_max_length => 'mbps_rd_max'],
1586         [bps_wr_max_length => 'mbps_wr_max'],
1587         [iops_max_length => 'iops_max'],
1588         [iops_rd_max_length => 'iops_rd_max'],
1589         [iops_wr_max_length => 'iops_wr_max']) {
1590         my ($option, $requires) = @$requirement;
1591         if ($res->{$option} && !$res->{$requires}) {
1592             warn "$option requires $requires\n";
1593             ++$error;
1594         }
1595     }
1596
1597     return undef if $error;
1598
1599     return undef if $res->{mbps_rd} && $res->{mbps};
1600     return undef if $res->{mbps_wr} && $res->{mbps};
1601     return undef if $res->{iops_rd} && $res->{iops};
1602     return undef if $res->{iops_wr} && $res->{iops};
1603
1604     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1605         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1606         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1607         return undef if $res->{interface} eq 'virtio';
1608     }
1609
1610     if (my $size = $res->{size}) {
1611         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1612     }
1613
1614     return $res;
1615 }
1616
1617 sub print_drive {
1618     my ($vmid, $drive) = @_;
1619     my $data = { %$drive };
1620     delete $data->{$_} for qw(index interface);
1621     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1622 }
1623
1624 sub scsi_inquiry {
1625     my($fh, $noerr) = @_;
1626
1627     my $SG_IO = 0x2285;
1628     my $SG_GET_VERSION_NUM = 0x2282;
1629
1630     my $versionbuf = "\x00" x 8;
1631     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1632     if (!$ret) {
1633         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1634         return undef;
1635     }
1636     my $version = unpack("I", $versionbuf);
1637     if ($version < 30000) {
1638         die "scsi generic interface too old\n"  if !$noerr;
1639         return undef;
1640     }
1641
1642     my $buf = "\x00" x 36;
1643     my $sensebuf = "\x00" x 8;
1644     my $cmd = pack("C x3 C x1", 0x12, 36);
1645
1646     # see /usr/include/scsi/sg.h
1647     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1648
1649     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1650                       length($sensebuf), 0, length($buf), $buf,
1651                       $cmd, $sensebuf, 6000);
1652
1653     $ret = ioctl($fh, $SG_IO, $packet);
1654     if (!$ret) {
1655         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1656         return undef;
1657     }
1658
1659     my @res = unpack($sg_io_hdr_t, $packet);
1660     if ($res[17] || $res[18]) {
1661         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1662         return undef;
1663     }
1664
1665     my $res = {};
1666     (my $byte0, my $byte1, $res->{vendor},
1667      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1668
1669     $res->{removable} = $byte1 & 128 ? 1 : 0;
1670     $res->{type} = $byte0 & 31;
1671
1672     return $res;
1673 }
1674
1675 sub path_is_scsi {
1676     my ($path) = @_;
1677
1678     my $fh = IO::File->new("+<$path") || return undef;
1679     my $res = scsi_inquiry($fh, 1);
1680     close($fh);
1681
1682     return $res;
1683 }
1684
1685 sub machine_type_is_q35 {
1686     my ($conf) = @_;
1687
1688     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1689 }
1690
1691 sub print_tabletdevice_full {
1692     my ($conf, $arch) = @_;
1693
1694     my $q35 = machine_type_is_q35($conf);
1695
1696     # we use uhci for old VMs because tablet driver was buggy in older qemu
1697     my $usbbus;
1698     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1699         $usbbus = 'ehci';
1700     } else {
1701         $usbbus = 'uhci';
1702     }
1703
1704     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1705 }
1706
1707 sub print_keyboarddevice_full {
1708     my ($conf, $arch, $machine) = @_;
1709
1710     return undef if $arch ne 'aarch64';
1711
1712     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1713 }
1714
1715 sub print_drivedevice_full {
1716     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1717
1718     my $device = '';
1719     my $maxdev = 0;
1720
1721     if ($drive->{interface} eq 'virtio') {
1722         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1723         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1724         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1725     } elsif ($drive->{interface} eq 'scsi') {
1726
1727         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1728         my $unit = $drive->{index} % $maxdev;
1729         my $devicetype = 'hd';
1730         my $path = '';
1731         if (drive_is_cdrom($drive)) {
1732             $devicetype = 'cd';
1733         } else {
1734             if ($drive->{file} =~ m|^/|) {
1735                 $path = $drive->{file};
1736                 if (my $info = path_is_scsi($path)) {
1737                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1738                         $devicetype = 'block';
1739                     } elsif ($info->{type} == 1) { # tape
1740                         $devicetype = 'generic';
1741                     }
1742                 }
1743             } else {
1744                  $path = PVE::Storage::path($storecfg, $drive->{file});
1745             }
1746
1747             if($path =~ m/^iscsi\:\/\//){
1748                 $devicetype = 'generic';
1749             }
1750         }
1751
1752         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1753             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1754         } else {
1755             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1756         }
1757
1758         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1759             $device .= ",rotation_rate=1";
1760         }
1761
1762     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1763         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1764         my $controller = int($drive->{index} / $maxdev);
1765         my $unit = $drive->{index} % $maxdev;
1766         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1767
1768         $device = "ide-$devicetype";
1769         if ($drive->{interface} eq 'ide') {
1770             $device .= ",bus=ide.$controller,unit=$unit";
1771         } else {
1772             $device .= ",bus=ahci$controller.$unit";
1773         }
1774         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1775
1776         if ($devicetype eq 'hd') {
1777             if (my $model = $drive->{model}) {
1778                 $model = URI::Escape::uri_unescape($model);
1779                 $device .= ",model=$model";
1780             }
1781             if ($drive->{ssd}) {
1782                 $device .= ",rotation_rate=1";
1783             }
1784         }
1785     } elsif ($drive->{interface} eq 'usb') {
1786         die "implement me";
1787         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1788     } else {
1789         die "unsupported interface type";
1790     }
1791
1792     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1793
1794     if (my $serial = $drive->{serial}) {
1795         $serial = URI::Escape::uri_unescape($serial);
1796         $device .= ",serial=$serial";
1797     }
1798
1799
1800     return $device;
1801 }
1802
1803 sub get_initiator_name {
1804     my $initiator;
1805
1806     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1807     while (defined(my $line = <$fh>)) {
1808         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1809         $initiator = $1;
1810         last;
1811     }
1812     $fh->close();
1813
1814     return $initiator;
1815 }
1816
1817 sub print_drive_full {
1818     my ($storecfg, $vmid, $drive) = @_;
1819
1820     my $path;
1821     my $volid = $drive->{file};
1822     my $format;
1823
1824     if (drive_is_cdrom($drive)) {
1825         $path = get_iso_path($storecfg, $vmid, $volid);
1826     } else {
1827         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1828         if ($storeid) {
1829             $path = PVE::Storage::path($storecfg, $volid);
1830             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1831             $format = qemu_img_format($scfg, $volname);
1832         } else {
1833             $path = $volid;
1834             $format = "raw";
1835         }
1836    }
1837
1838     my $opts = '';
1839     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1840     foreach my $o (@qemu_drive_options) {
1841         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1842     }
1843
1844     # snapshot only accepts on|off
1845     if (defined($drive->{snapshot})) {
1846         my $v = $drive->{snapshot} ? 'on' : 'off';
1847         $opts .= ",snapshot=$v";
1848     }
1849
1850     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1851         my ($dir, $qmpname) = @$type;
1852         if (my $v = $drive->{"mbps$dir"}) {
1853             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1854         }
1855         if (my $v = $drive->{"mbps${dir}_max"}) {
1856             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1857         }
1858         if (my $v = $drive->{"bps${dir}_max_length"}) {
1859             $opts .= ",throttling.bps$qmpname-max-length=$v";
1860         }
1861         if (my $v = $drive->{"iops${dir}"}) {
1862             $opts .= ",throttling.iops$qmpname=$v";
1863         }
1864         if (my $v = $drive->{"iops${dir}_max"}) {
1865             $opts .= ",throttling.iops$qmpname-max=$v";
1866         }
1867         if (my $v = $drive->{"iops${dir}_max_length"}) {
1868             $opts .= ",throttling.iops$qmpname-max-length=$v";
1869         }
1870     }
1871
1872     $opts .= ",format=$format" if $format && !$drive->{format};
1873
1874     my $cache_direct = 0;
1875
1876     if (my $cache = $drive->{cache}) {
1877         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         $opts .= ",cache=none";
1880         $cache_direct = 1;
1881     }
1882
1883     # aio native works only with O_DIRECT
1884     if (!$drive->{aio}) {
1885         if($cache_direct) {
1886             $opts .= ",aio=native";
1887         } else {
1888             $opts .= ",aio=threads";
1889         }
1890     }
1891
1892     if (!drive_is_cdrom($drive)) {
1893         my $detectzeroes;
1894         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1895             $detectzeroes = 'off';
1896         } elsif ($drive->{discard}) {
1897             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1898         } else {
1899             # This used to be our default with discard not being specified:
1900             $detectzeroes = 'on';
1901         }
1902         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1903     }
1904
1905     my $pathinfo = $path ? "file=$path," : '';
1906
1907     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1908 }
1909
1910 sub print_netdevice_full {
1911     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1912
1913     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1914
1915     my $device = $net->{model};
1916     if ($net->{model} eq 'virtio') {
1917          $device = 'virtio-net-pci';
1918      };
1919
1920     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1921     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1922     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1923         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1924         my $vectors = $net->{queues} * 2 + 2;
1925         $tmpstr .= ",vectors=$vectors,mq=on";
1926     }
1927     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1928
1929     if ($use_old_bios_files) {
1930         my $romfile;
1931         if ($device eq 'virtio-net-pci') {
1932             $romfile = 'pxe-virtio.rom';
1933         } elsif ($device eq 'e1000') {
1934             $romfile = 'pxe-e1000.rom';
1935         } elsif ($device eq 'ne2k') {
1936             $romfile = 'pxe-ne2k_pci.rom';
1937         } elsif ($device eq 'pcnet') {
1938             $romfile = 'pxe-pcnet.rom';
1939         } elsif ($device eq 'rtl8139') {
1940             $romfile = 'pxe-rtl8139.rom';
1941         }
1942         $tmpstr .= ",romfile=$romfile" if $romfile;
1943     }
1944
1945     return $tmpstr;
1946 }
1947
1948 sub print_netdev_full {
1949     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1950
1951     my $i = '';
1952     if ($netid =~ m/^net(\d+)$/) {
1953         $i = int($1);
1954     }
1955
1956     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1957
1958     my $ifname = "tap${vmid}i$i";
1959
1960     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1961     die "interface name '$ifname' is too long (max 15 character)\n"
1962         if length($ifname) >= 16;
1963
1964     my $vhostparam = '';
1965     if (is_native($arch)) {
1966         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1967     }
1968
1969     my $vmname = $conf->{name} || "vm$vmid";
1970
1971     my $netdev = "";
1972     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1973
1974     if ($net->{bridge}) {
1975         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1976     } else {
1977         $netdev = "type=user,id=$netid,hostname=$vmname";
1978     }
1979
1980     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1981
1982     return $netdev;
1983 }
1984
1985
1986 sub print_cpu_device {
1987     my ($conf, $id) = @_;
1988
1989     my $kvm = $conf->{kvm} // 1;
1990     my $cpu = $kvm ? "kvm64" : "qemu64";
1991     if (my $cputype = $conf->{cpu}) {
1992         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1993             or die "Cannot parse cpu description: $cputype\n";
1994         $cpu = $cpuconf->{cputype};
1995     }
1996
1997     my $cores = $conf->{cores} || 1;
1998
1999     my $current_core = ($id - 1) % $cores;
2000     my $current_socket = int(($id - 1 - $current_core)/$cores);
2001
2002     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2003 }
2004
2005 my $vga_map = {
2006     'cirrus' => 'cirrus-vga',
2007     'std' => 'VGA',
2008     'vmware' => 'vmware-svga',
2009     'virtio' => 'virtio-vga',
2010 };
2011
2012 sub print_vga_device {
2013     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2014
2015     my $type = $vga_map->{$vga->{type}};
2016     if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
2017         $type = 'virtio-gpu';
2018     }
2019     my $vgamem_mb = $vga->{memory};
2020     if ($qxlnum) {
2021         $type = $id ? 'qxl' : 'qxl-vga';
2022     }
2023     die "no devicetype for $vga->{type}\n" if !$type;
2024
2025     my $memory = "";
2026     if ($vgamem_mb) {
2027         if ($vga->{type} eq 'virtio') {
2028             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2029             $memory = ",max_hostmem=$bytes";
2030         } elsif ($qxlnum) {
2031             # from https://www.spice-space.org/multiple-monitors.html
2032             $memory = ",vgamem_mb=$vga->{memory}";
2033             my $ram = $vgamem_mb * 4;
2034             my $vram = $vgamem_mb * 2;
2035             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2036         } else {
2037             $memory = ",vgamem_mb=$vga->{memory}";
2038         }
2039     } elsif ($qxlnum && $id) {
2040         $memory = ",ram_size=67108864,vram_size=33554432";
2041     }
2042
2043     my $q35 = machine_type_is_q35($conf);
2044     my $vgaid = "vga" . ($id // '');
2045     my $pciaddr;
2046
2047     if ($q35 && $vgaid eq 'vga') {
2048         # the first display uses pcie.0 bus on q35 machines
2049         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2050     } else {
2051         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2052     }
2053
2054     return "$type,id=${vgaid}${memory}${pciaddr}";
2055 }
2056
2057 sub drive_is_cloudinit {
2058     my ($drive) = @_;
2059     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2060 }
2061
2062 sub drive_is_cdrom {
2063     my ($drive, $exclude_cloudinit) = @_;
2064
2065     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2066
2067     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2068
2069 }
2070
2071 sub parse_number_sets {
2072     my ($set) = @_;
2073     my $res = [];
2074     foreach my $part (split(/;/, $set)) {
2075         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2076             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2077             push @$res, [ $1, $2 ];
2078         } else {
2079             die "invalid range: $part\n";
2080         }
2081     }
2082     return $res;
2083 }
2084
2085 sub parse_numa {
2086     my ($data) = @_;
2087
2088     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2089     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2090     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2091     return $res;
2092 }
2093
2094 sub parse_hostpci {
2095     my ($value) = @_;
2096
2097     return undef if !$value;
2098
2099     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2100
2101     my @idlist = split(/;/, $res->{host});
2102     delete $res->{host};
2103     foreach my $id (@idlist) {
2104         if ($id =~ /^$PCIRE$/) {
2105             if (defined($2)) {
2106                 push @{$res->{pciid}}, { id => $1, function => $2 };
2107             } else {
2108                 my $pcidevices = PVE::SysFSTools::lspci($1);
2109                 $res->{pciid} = $pcidevices->{$1};
2110             }
2111         } else {
2112             # should have been caught by parse_property_string already
2113             die "failed to parse PCI id: $id\n";
2114         }
2115     }
2116     return $res;
2117 }
2118
2119 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2120 sub parse_net {
2121     my ($data) = @_;
2122
2123     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2124     if ($@) {
2125         warn $@;
2126         return undef;
2127     }
2128     if (!defined($res->{macaddr})) {
2129         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2130         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2131     }
2132     return $res;
2133 }
2134
2135 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2136 sub parse_ipconfig {
2137     my ($data) = @_;
2138
2139     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2140     if ($@) {
2141         warn $@;
2142         return undef;
2143     }
2144
2145     if ($res->{gw} && !$res->{ip}) {
2146         warn 'gateway specified without specifying an IP address';
2147         return undef;
2148     }
2149     if ($res->{gw6} && !$res->{ip6}) {
2150         warn 'IPv6 gateway specified without specifying an IPv6 address';
2151         return undef;
2152     }
2153     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2154         warn 'gateway specified together with DHCP';
2155         return undef;
2156     }
2157     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2158         # gw6 + auto/dhcp
2159         warn "IPv6 gateway specified together with $res->{ip6} address";
2160         return undef;
2161     }
2162
2163     if (!$res->{ip} && !$res->{ip6}) {
2164         return { ip => 'dhcp', ip6 => 'dhcp' };
2165     }
2166
2167     return $res;
2168 }
2169
2170 sub print_net {
2171     my $net = shift;
2172
2173     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2174 }
2175
2176 sub add_random_macs {
2177     my ($settings) = @_;
2178
2179     foreach my $opt (keys %$settings) {
2180         next if $opt !~ m/^net(\d+)$/;
2181         my $net = parse_net($settings->{$opt});
2182         next if !$net;
2183         $settings->{$opt} = print_net($net);
2184     }
2185 }
2186
2187 sub vm_is_volid_owner {
2188     my ($storecfg, $vmid, $volid) = @_;
2189
2190     if ($volid !~  m|^/|) {
2191         my ($path, $owner);
2192         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2193         if ($owner && ($owner == $vmid)) {
2194             return 1;
2195         }
2196     }
2197
2198     return undef;
2199 }
2200
2201 sub split_flagged_list {
2202     my $text = shift || '';
2203     $text =~ s/[,;]/ /g;
2204     $text =~ s/^\s+//;
2205     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2206 }
2207
2208 sub join_flagged_list {
2209     my ($how, $lst) = @_;
2210     join $how, map { $lst->{$_} . $_ } keys %$lst;
2211 }
2212
2213 sub vmconfig_delete_pending_option {
2214     my ($conf, $key, $force) = @_;
2215
2216     delete $conf->{pending}->{$key};
2217     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2218     $pending_delete_hash->{$key} = $force ? '!' : '';
2219     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2220 }
2221
2222 sub vmconfig_undelete_pending_option {
2223     my ($conf, $key) = @_;
2224
2225     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2226     delete $pending_delete_hash->{$key};
2227
2228     if (%$pending_delete_hash) {
2229         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2230     } else {
2231         delete $conf->{pending}->{delete};
2232     }
2233 }
2234
2235 sub vmconfig_register_unused_drive {
2236     my ($storecfg, $vmid, $conf, $drive) = @_;
2237
2238     if (drive_is_cloudinit($drive)) {
2239         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2240         warn $@ if $@;
2241     } elsif (!drive_is_cdrom($drive)) {
2242         my $volid = $drive->{file};
2243         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2244             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2245         }
2246     }
2247 }
2248
2249 sub vmconfig_cleanup_pending {
2250     my ($conf) = @_;
2251
2252     # remove pending changes when nothing changed
2253     my $changes;
2254     foreach my $opt (keys %{$conf->{pending}}) {
2255         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2256             $changes = 1;
2257             delete $conf->{pending}->{$opt};
2258         }
2259     }
2260
2261     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2262     my $pending_delete_hash = {};
2263     while (my ($opt, $force) = each %$current_delete_hash) {
2264         if (defined($conf->{$opt})) {
2265             $pending_delete_hash->{$opt} = $force;
2266         } else {
2267             $changes = 1;
2268         }
2269     }
2270
2271     if (%$pending_delete_hash) {
2272         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2273     } else {
2274         delete $conf->{pending}->{delete};
2275     }
2276
2277     return $changes;
2278 }
2279
2280 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2281 my $smbios1_fmt = {
2282     uuid => {
2283         type => 'string',
2284         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2285         format_description => 'UUID',
2286         description => "Set SMBIOS1 UUID.",
2287         optional => 1,
2288     },
2289     version => {
2290         type => 'string',
2291         pattern => '\S+',
2292         format_description => 'string',
2293         description => "Set SMBIOS1 version.",
2294         optional => 1,
2295     },
2296     serial => {
2297         type => 'string',
2298         pattern => '\S+',
2299         format_description => 'string',
2300         description => "Set SMBIOS1 serial number.",
2301         optional => 1,
2302     },
2303     manufacturer => {
2304         type => 'string',
2305         pattern => '\S+',
2306         format_description => 'string',
2307         description => "Set SMBIOS1 manufacturer.",
2308         optional => 1,
2309     },
2310     product => {
2311         type => 'string',
2312         pattern => '\S+',
2313         format_description => 'string',
2314         description => "Set SMBIOS1 product ID.",
2315         optional => 1,
2316     },
2317     sku => {
2318         type => 'string',
2319         pattern => '\S+',
2320         format_description => 'string',
2321         description => "Set SMBIOS1 SKU string.",
2322         optional => 1,
2323     },
2324     family => {
2325         type => 'string',
2326         pattern => '\S+',
2327         format_description => 'string',
2328         description => "Set SMBIOS1 family string.",
2329         optional => 1,
2330     },
2331 };
2332
2333 sub parse_smbios1 {
2334     my ($data) = @_;
2335
2336     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2337     warn $@ if $@;
2338     return $res;
2339 }
2340
2341 sub print_smbios1 {
2342     my ($smbios1) = @_;
2343     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2344 }
2345
2346 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2347
2348 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2349 sub verify_bootdisk {
2350     my ($value, $noerr) = @_;
2351
2352     return $value if is_valid_drivename($value);
2353
2354     return undef if $noerr;
2355
2356     die "invalid boot disk '$value'\n";
2357 }
2358
2359 sub parse_watchdog {
2360     my ($value) = @_;
2361
2362     return undef if !$value;
2363
2364     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2365     warn $@ if $@;
2366     return $res;
2367 }
2368
2369 sub parse_guest_agent {
2370     my ($value) = @_;
2371
2372     return {} if !defined($value->{agent});
2373
2374     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2375     warn $@ if $@;
2376
2377     # if the agent is disabled ignore the other potentially set properties
2378     return {} if !$res->{enabled};
2379     return $res;
2380 }
2381
2382 sub parse_vga {
2383     my ($value) = @_;
2384
2385     return {} if !$value;
2386     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2387     warn $@ if $@;
2388     return $res;
2389 }
2390
2391 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2392 sub verify_usb_device {
2393     my ($value, $noerr) = @_;
2394
2395     return $value if parse_usb_device($value);
2396
2397     return undef if $noerr;
2398
2399     die "unable to parse usb device\n";
2400 }
2401
2402 # add JSON properties for create and set function
2403 sub json_config_properties {
2404     my $prop = shift;
2405
2406     foreach my $opt (keys %$confdesc) {
2407         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2408         $prop->{$opt} = $confdesc->{$opt};
2409     }
2410
2411     return $prop;
2412 }
2413
2414 # return copy of $confdesc_cloudinit to generate documentation
2415 sub cloudinit_config_properties {
2416
2417     return dclone($confdesc_cloudinit);
2418 }
2419
2420 sub check_type {
2421     my ($key, $value) = @_;
2422
2423     die "unknown setting '$key'\n" if !$confdesc->{$key};
2424
2425     my $type = $confdesc->{$key}->{type};
2426
2427     if (!defined($value)) {
2428         die "got undefined value\n";
2429     }
2430
2431     if ($value =~ m/[\n\r]/) {
2432         die "property contains a line feed\n";
2433     }
2434
2435     if ($type eq 'boolean') {
2436         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2437         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2438         die "type check ('boolean') failed - got '$value'\n";
2439     } elsif ($type eq 'integer') {
2440         return int($1) if $value =~ m/^(\d+)$/;
2441         die "type check ('integer') failed - got '$value'\n";
2442     } elsif ($type eq 'number') {
2443         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2444         die "type check ('number') failed - got '$value'\n";
2445     } elsif ($type eq 'string') {
2446         if (my $fmt = $confdesc->{$key}->{format}) {
2447             PVE::JSONSchema::check_format($fmt, $value);
2448             return $value;
2449         }
2450         $value =~ s/^\"(.*)\"$/$1/;
2451         return $value;
2452     } else {
2453         die "internal error"
2454     }
2455 }
2456
2457 sub touch_config {
2458     my ($vmid) = @_;
2459
2460     my $conf = PVE::QemuConfig->config_file($vmid);
2461     utime undef, undef, $conf;
2462 }
2463
2464 sub destroy_vm {
2465     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2466
2467     my $conffile = PVE::QemuConfig->config_file($vmid);
2468
2469     my $conf = PVE::QemuConfig->load_config($vmid);
2470
2471     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2472
2473     if ($conf->{template}) {
2474         # check if any base image is still used by a linked clone
2475         foreach_drive($conf, sub {
2476                 my ($ds, $drive) = @_;
2477
2478                 return if drive_is_cdrom($drive);
2479
2480                 my $volid = $drive->{file};
2481
2482                 return if !$volid || $volid =~ m|^/|;
2483
2484                 die "base volume '$volid' is still in use by linked cloned\n"
2485                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2486
2487         });
2488     }
2489
2490     # only remove disks owned by this VM
2491     foreach_drive($conf, sub {
2492         my ($ds, $drive) = @_;
2493
2494         return if drive_is_cdrom($drive, 1);
2495
2496         my $volid = $drive->{file};
2497
2498         return if !$volid || $volid =~ m|^/|;
2499
2500         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2501         return if !$path || !$owner || ($owner != $vmid);
2502
2503         eval {
2504             PVE::Storage::vdisk_free($storecfg, $volid);
2505         };
2506         warn "Could not remove disk '$volid', check manually: $@" if $@;
2507
2508     });
2509
2510     if ($keep_empty_config) {
2511         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2512     } else {
2513         unlink $conffile;
2514     }
2515
2516     # also remove unused disk
2517     eval {
2518         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2519
2520         eval {
2521             PVE::Storage::foreach_volid($dl, sub {
2522                 my ($volid, $sid, $volname, $d) = @_;
2523                 PVE::Storage::vdisk_free($storecfg, $volid);
2524             });
2525         };
2526         warn $@ if $@;
2527
2528     };
2529     warn $@ if $@;
2530 }
2531
2532 sub parse_vm_config {
2533     my ($filename, $raw) = @_;
2534
2535     return undef if !defined($raw);
2536
2537     my $res = {
2538         digest => Digest::SHA::sha1_hex($raw),
2539         snapshots => {},
2540         pending => {},
2541     };
2542
2543     $filename =~ m|/qemu-server/(\d+)\.conf$|
2544         || die "got strange filename '$filename'";
2545
2546     my $vmid = $1;
2547
2548     my $conf = $res;
2549     my $descr;
2550     my $section = '';
2551
2552     my @lines = split(/\n/, $raw);
2553     foreach my $line (@lines) {
2554         next if $line =~ m/^\s*$/;
2555
2556         if ($line =~ m/^\[PENDING\]\s*$/i) {
2557             $section = 'pending';
2558             if (defined($descr)) {
2559                 $descr =~ s/\s+$//;
2560                 $conf->{description} = $descr;
2561             }
2562             $descr = undef;
2563             $conf = $res->{$section} = {};
2564             next;
2565
2566         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2567             $section = $1;
2568             if (defined($descr)) {
2569                 $descr =~ s/\s+$//;
2570                 $conf->{description} = $descr;
2571             }
2572             $descr = undef;
2573             $conf = $res->{snapshots}->{$section} = {};
2574             next;
2575         }
2576
2577         if ($line =~ m/^\#(.*)\s*$/) {
2578             $descr = '' if !defined($descr);
2579             $descr .= PVE::Tools::decode_text($1) . "\n";
2580             next;
2581         }
2582
2583         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2584             $descr = '' if !defined($descr);
2585             $descr .= PVE::Tools::decode_text($2);
2586         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2587             $conf->{snapstate} = $1;
2588         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2589             my $key = $1;
2590             my $value = $2;
2591             $conf->{$key} = $value;
2592         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2593             my $value = $1;
2594             if ($section eq 'pending') {
2595                 $conf->{delete} = $value; # we parse this later
2596             } else {
2597                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2598             }
2599         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2600             my $key = $1;
2601             my $value = $2;
2602             eval { $value = check_type($key, $value); };
2603             if ($@) {
2604                 warn "vm $vmid - unable to parse value of '$key' - $@";
2605             } else {
2606                 $key = 'ide2' if $key eq 'cdrom';
2607                 my $fmt = $confdesc->{$key}->{format};
2608                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2609                     my $v = parse_drive($key, $value);
2610                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2611                         $v->{file} = $volid;
2612                         $value = print_drive($vmid, $v);
2613                     } else {
2614                         warn "vm $vmid - unable to parse value of '$key'\n";
2615                         next;
2616                     }
2617                 }
2618
2619                 $conf->{$key} = $value;
2620             }
2621         }
2622     }
2623
2624     if (defined($descr)) {
2625         $descr =~ s/\s+$//;
2626         $conf->{description} = $descr;
2627     }
2628     delete $res->{snapstate}; # just to be sure
2629
2630     return $res;
2631 }
2632
2633 sub write_vm_config {
2634     my ($filename, $conf) = @_;
2635
2636     delete $conf->{snapstate}; # just to be sure
2637
2638     if ($conf->{cdrom}) {
2639         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2640         $conf->{ide2} = $conf->{cdrom};
2641         delete $conf->{cdrom};
2642     }
2643
2644     # we do not use 'smp' any longer
2645     if ($conf->{sockets}) {
2646         delete $conf->{smp};
2647     } elsif ($conf->{smp}) {
2648         $conf->{sockets} = $conf->{smp};
2649         delete $conf->{cores};
2650         delete $conf->{smp};
2651     }
2652
2653     my $used_volids = {};
2654
2655     my $cleanup_config = sub {
2656         my ($cref, $pending, $snapname) = @_;
2657
2658         foreach my $key (keys %$cref) {
2659             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2660                 $key eq 'snapstate' || $key eq 'pending';
2661             my $value = $cref->{$key};
2662             if ($key eq 'delete') {
2663                 die "propertry 'delete' is only allowed in [PENDING]\n"
2664                     if !$pending;
2665                 # fixme: check syntax?
2666                 next;
2667             }
2668             eval { $value = check_type($key, $value); };
2669             die "unable to parse value of '$key' - $@" if $@;
2670
2671             $cref->{$key} = $value;
2672
2673             if (!$snapname && is_valid_drivename($key)) {
2674                 my $drive = parse_drive($key, $value);
2675                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2676             }
2677         }
2678     };
2679
2680     &$cleanup_config($conf);
2681
2682     &$cleanup_config($conf->{pending}, 1);
2683
2684     foreach my $snapname (keys %{$conf->{snapshots}}) {
2685         die "internal error" if $snapname eq 'pending';
2686         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2687     }
2688
2689     # remove 'unusedX' settings if we re-add a volume
2690     foreach my $key (keys %$conf) {
2691         my $value = $conf->{$key};
2692         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2693             delete $conf->{$key};
2694         }
2695     }
2696
2697     my $generate_raw_config = sub {
2698         my ($conf, $pending) = @_;
2699
2700         my $raw = '';
2701
2702         # add description as comment to top of file
2703         if (defined(my $descr = $conf->{description})) {
2704             if ($descr) {
2705                 foreach my $cl (split(/\n/, $descr)) {
2706                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2707                 }
2708             } else {
2709                 $raw .= "#\n" if $pending;
2710             }
2711         }
2712
2713         foreach my $key (sort keys %$conf) {
2714             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2715             $raw .= "$key: $conf->{$key}\n";
2716         }
2717         return $raw;
2718     };
2719
2720     my $raw = &$generate_raw_config($conf);
2721
2722     if (scalar(keys %{$conf->{pending}})){
2723         $raw .= "\n[PENDING]\n";
2724         $raw .= &$generate_raw_config($conf->{pending}, 1);
2725     }
2726
2727     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2728         $raw .= "\n[$snapname]\n";
2729         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2730     }
2731
2732     return $raw;
2733 }
2734
2735 sub load_defaults {
2736
2737     my $res = {};
2738
2739     # we use static defaults from our JSON schema configuration
2740     foreach my $key (keys %$confdesc) {
2741         if (defined(my $default = $confdesc->{$key}->{default})) {
2742             $res->{$key} = $default;
2743         }
2744     }
2745
2746     return $res;
2747 }
2748
2749 sub config_list {
2750     my $vmlist = PVE::Cluster::get_vmlist();
2751     my $res = {};
2752     return $res if !$vmlist || !$vmlist->{ids};
2753     my $ids = $vmlist->{ids};
2754
2755     foreach my $vmid (keys %$ids) {
2756         my $d = $ids->{$vmid};
2757         next if !$d->{node} || $d->{node} ne $nodename;
2758         next if !$d->{type} || $d->{type} ne 'qemu';
2759         $res->{$vmid}->{exists} = 1;
2760     }
2761     return $res;
2762 }
2763
2764 # test if VM uses local resources (to prevent migration)
2765 sub check_local_resources {
2766     my ($conf, $noerr) = @_;
2767
2768     my $loc_res = 0;
2769
2770     $loc_res = 1 if $conf->{hostusb}; # old syntax
2771     $loc_res = 1 if $conf->{hostpci}; # old syntax
2772
2773     foreach my $k (keys %$conf) {
2774         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2775         # sockets are safe: they will recreated be on the target side post-migrate
2776         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2777         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2778     }
2779
2780     die "VM uses local resources\n" if $loc_res && !$noerr;
2781
2782     return $loc_res;
2783 }
2784
2785 # check if used storages are available on all nodes (use by migrate)
2786 sub check_storage_availability {
2787     my ($storecfg, $conf, $node) = @_;
2788
2789     foreach_drive($conf, sub {
2790         my ($ds, $drive) = @_;
2791
2792         my $volid = $drive->{file};
2793         return if !$volid;
2794
2795         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2796         return if !$sid;
2797
2798         # check if storage is available on both nodes
2799         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2800         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2801    });
2802 }
2803
2804 # list nodes where all VM images are available (used by has_feature API)
2805 sub shared_nodes {
2806     my ($conf, $storecfg) = @_;
2807
2808     my $nodelist = PVE::Cluster::get_nodelist();
2809     my $nodehash = { map { $_ => 1 } @$nodelist };
2810     my $nodename = PVE::INotify::nodename();
2811
2812     foreach_drive($conf, sub {
2813         my ($ds, $drive) = @_;
2814
2815         my $volid = $drive->{file};
2816         return if !$volid;
2817
2818         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2819         if ($storeid) {
2820             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2821             if ($scfg->{disable}) {
2822                 $nodehash = {};
2823             } elsif (my $avail = $scfg->{nodes}) {
2824                 foreach my $node (keys %$nodehash) {
2825                     delete $nodehash->{$node} if !$avail->{$node};
2826                 }
2827             } elsif (!$scfg->{shared}) {
2828                 foreach my $node (keys %$nodehash) {
2829                     delete $nodehash->{$node} if $node ne $nodename
2830                 }
2831             }
2832         }
2833     });
2834
2835     return $nodehash
2836 }
2837
2838 sub check_cmdline {
2839     my ($pidfile, $pid) = @_;
2840
2841     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2842     if (defined($fh)) {
2843         my $line = <$fh>;
2844         $fh->close;
2845         return undef if !$line;
2846         my @param = split(/\0/, $line);
2847
2848         my $cmd = $param[0];
2849         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2850
2851         for (my $i = 0; $i < scalar (@param); $i++) {
2852             my $p = $param[$i];
2853             next if !$p;
2854             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2855                 my $p = $param[$i+1];
2856                 return 1 if $p && ($p eq $pidfile);
2857                 return undef;
2858             }
2859         }
2860     }
2861     return undef;
2862 }
2863
2864 sub check_running {
2865     my ($vmid, $nocheck, $node) = @_;
2866
2867     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2868
2869     die "unable to find configuration file for VM $vmid - no such machine\n"
2870         if !$nocheck && ! -f $filename;
2871
2872     my $pidfile = pidfile_name($vmid);
2873
2874     if (my $fd = IO::File->new("<$pidfile")) {
2875         my $st = stat($fd);
2876         my $line = <$fd>;
2877         close($fd);
2878
2879         my $mtime = $st->mtime;
2880         if ($mtime > time()) {
2881             warn "file '$filename' modified in future\n";
2882         }
2883
2884         if ($line =~ m/^(\d+)$/) {
2885             my $pid = $1;
2886             if (check_cmdline($pidfile, $pid)) {
2887                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2888                     return $pid;
2889                 }
2890             }
2891         }
2892     }
2893
2894     return undef;
2895 }
2896
2897 sub vzlist {
2898
2899     my $vzlist = config_list();
2900
2901     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2902
2903     while (defined(my $de = $fd->read)) {
2904         next if $de !~ m/^(\d+)\.pid$/;
2905         my $vmid = $1;
2906         next if !defined($vzlist->{$vmid});
2907         if (my $pid = check_running($vmid)) {
2908             $vzlist->{$vmid}->{pid} = $pid;
2909         }
2910     }
2911
2912     return $vzlist;
2913 }
2914
2915 sub disksize {
2916     my ($storecfg, $conf) = @_;
2917
2918     my $bootdisk = $conf->{bootdisk};
2919     return undef if !$bootdisk;
2920     return undef if !is_valid_drivename($bootdisk);
2921
2922     return undef if !$conf->{$bootdisk};
2923
2924     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2925     return undef if !defined($drive);
2926
2927     return undef if drive_is_cdrom($drive);
2928
2929     my $volid = $drive->{file};
2930     return undef if !$volid;
2931
2932     return $drive->{size};
2933 }
2934
2935 our $vmstatus_return_properties = {
2936     vmid => get_standard_option('pve-vmid'),
2937     status => {
2938         description => "Qemu process status.",
2939         type => 'string',
2940         enum => ['stopped', 'running'],
2941     },
2942     maxmem => {
2943         description => "Maximum memory in bytes.",
2944         type => 'integer',
2945         optional => 1,
2946         renderer => 'bytes',
2947     },
2948     maxdisk => {
2949         description => "Root disk size in bytes.",
2950         type => 'integer',
2951         optional => 1,
2952         renderer => 'bytes',
2953     },
2954     name => {
2955         description => "VM name.",
2956         type => 'string',
2957         optional => 1,
2958     },
2959     qmpstatus => {
2960         description => "Qemu QMP agent status.",
2961         type => 'string',
2962         optional => 1,
2963     },
2964     pid => {
2965         description => "PID of running qemu process.",
2966         type => 'integer',
2967         optional => 1,
2968     },
2969     uptime => {
2970         description => "Uptime.",
2971         type => 'integer',
2972         optional => 1,
2973         renderer => 'duration',
2974     },
2975     cpus => {
2976         description => "Maximum usable CPUs.",
2977         type => 'number',
2978         optional => 1,
2979     },
2980 };
2981
2982 my $last_proc_pid_stat;
2983
2984 # get VM status information
2985 # This must be fast and should not block ($full == false)
2986 # We only query KVM using QMP if $full == true (this can be slow)
2987 sub vmstatus {
2988     my ($opt_vmid, $full) = @_;
2989
2990     my $res = {};
2991
2992     my $storecfg = PVE::Storage::config();
2993
2994     my $list = vzlist();
2995     my $defaults = load_defaults();
2996
2997     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2998
2999     my $cpucount = $cpuinfo->{cpus} || 1;
3000
3001     foreach my $vmid (keys %$list) {
3002         next if $opt_vmid && ($vmid ne $opt_vmid);
3003
3004         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3005         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3006
3007         my $d = { vmid => $vmid };
3008         $d->{pid} = $list->{$vmid}->{pid};
3009
3010         # fixme: better status?
3011         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3012
3013         my $size = disksize($storecfg, $conf);
3014         if (defined($size)) {
3015             $d->{disk} = 0; # no info available
3016             $d->{maxdisk} = $size;
3017         } else {
3018             $d->{disk} = 0;
3019             $d->{maxdisk} = 0;
3020         }
3021
3022         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3023             * ($conf->{cores} || $defaults->{cores});
3024         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3025         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3026
3027         $d->{name} = $conf->{name} || "VM $vmid";
3028         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3029             : $defaults->{memory}*(1024*1024);
3030
3031         if ($conf->{balloon}) {
3032             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3033             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3034                 : $defaults->{shares};
3035         }
3036
3037         $d->{uptime} = 0;
3038         $d->{cpu} = 0;
3039         $d->{mem} = 0;
3040
3041         $d->{netout} = 0;
3042         $d->{netin} = 0;
3043
3044         $d->{diskread} = 0;
3045         $d->{diskwrite} = 0;
3046
3047         $d->{template} = PVE::QemuConfig->is_template($conf);
3048
3049         $d->{serial} = 1 if conf_has_serial($conf);
3050
3051         $res->{$vmid} = $d;
3052     }
3053
3054     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3055     foreach my $dev (keys %$netdev) {
3056         next if $dev !~ m/^tap([1-9]\d*)i/;
3057         my $vmid = $1;
3058         my $d = $res->{$vmid};
3059         next if !$d;
3060
3061         $d->{netout} += $netdev->{$dev}->{receive};
3062         $d->{netin} += $netdev->{$dev}->{transmit};
3063
3064         if ($full) {
3065             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3066             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3067         }
3068
3069     }
3070
3071     my $ctime = gettimeofday;
3072
3073     foreach my $vmid (keys %$list) {
3074
3075         my $d = $res->{$vmid};
3076         my $pid = $d->{pid};
3077         next if !$pid;
3078
3079         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3080         next if !$pstat; # not running
3081
3082         my $used = $pstat->{utime} + $pstat->{stime};
3083
3084         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3085
3086         if ($pstat->{vsize}) {
3087             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3088         }
3089
3090         my $old = $last_proc_pid_stat->{$pid};
3091         if (!$old) {
3092             $last_proc_pid_stat->{$pid} = {
3093                 time => $ctime,
3094                 used => $used,
3095                 cpu => 0,
3096             };
3097             next;
3098         }
3099
3100         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3101
3102         if ($dtime > 1000) {
3103             my $dutime = $used -  $old->{used};
3104
3105             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3106             $last_proc_pid_stat->{$pid} = {
3107                 time => $ctime,
3108                 used => $used,
3109                 cpu => $d->{cpu},
3110             };
3111         } else {
3112             $d->{cpu} = $old->{cpu};
3113         }
3114     }
3115
3116     return $res if !$full;
3117
3118     my $qmpclient = PVE::QMPClient->new();
3119
3120     my $ballooncb = sub {
3121         my ($vmid, $resp) = @_;
3122
3123         my $info = $resp->{'return'};
3124         return if !$info->{max_mem};
3125
3126         my $d = $res->{$vmid};
3127
3128         # use memory assigned to VM
3129         $d->{maxmem} = $info->{max_mem};
3130         $d->{balloon} = $info->{actual};
3131
3132         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3133             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3134             $d->{freemem} = $info->{free_mem};
3135         }
3136
3137         $d->{ballooninfo} = $info;
3138     };
3139
3140     my $blockstatscb = sub {
3141         my ($vmid, $resp) = @_;
3142         my $data = $resp->{'return'} || [];
3143         my $totalrdbytes = 0;
3144         my $totalwrbytes = 0;
3145
3146         for my $blockstat (@$data) {
3147             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3148             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3149
3150             $blockstat->{device} =~ s/drive-//;
3151             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3152         }
3153         $res->{$vmid}->{diskread} = $totalrdbytes;
3154         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3155     };
3156
3157     my $statuscb = sub {
3158         my ($vmid, $resp) = @_;
3159
3160         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3161         # this fails if ballon driver is not loaded, so this must be
3162         # the last commnand (following command are aborted if this fails).
3163         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3164
3165         my $status = 'unknown';
3166         if (!defined($status = $resp->{'return'}->{status})) {
3167             warn "unable to get VM status\n";
3168             return;
3169         }
3170
3171         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3172     };
3173
3174     foreach my $vmid (keys %$list) {
3175         next if $opt_vmid && ($vmid ne $opt_vmid);
3176         next if !$res->{$vmid}->{pid}; # not running
3177         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3178     }
3179
3180     $qmpclient->queue_execute(undef, 2);
3181
3182     foreach my $vmid (keys %$list) {
3183         next if $opt_vmid && ($vmid ne $opt_vmid);
3184         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3185     }
3186
3187     return $res;
3188 }
3189
3190 sub foreach_drive {
3191     my ($conf, $func, @param) = @_;
3192
3193     foreach my $ds (valid_drive_names()) {
3194         next if !defined($conf->{$ds});
3195
3196         my $drive = parse_drive($ds, $conf->{$ds});
3197         next if !$drive;
3198
3199         &$func($ds, $drive, @param);
3200     }
3201 }
3202
3203 sub foreach_volid {
3204     my ($conf, $func, @param) = @_;
3205
3206     my $volhash = {};
3207
3208     my $test_volid = sub {
3209         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3210
3211         return if !$volid;
3212
3213         $volhash->{$volid}->{cdrom} //= 1;
3214         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3215
3216         $volhash->{$volid}->{replicate} //= 0;
3217         $volhash->{$volid}->{replicate} = 1 if $replicate;
3218
3219         $volhash->{$volid}->{shared} //= 0;
3220         $volhash->{$volid}->{shared} = 1 if $shared;
3221
3222         $volhash->{$volid}->{referenced_in_config} //= 0;
3223         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3224
3225         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3226             if defined($snapname);
3227     };
3228
3229     foreach_drive($conf, sub {
3230         my ($ds, $drive) = @_;
3231         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3232     });
3233
3234     foreach my $snapname (keys %{$conf->{snapshots}}) {
3235         my $snap = $conf->{snapshots}->{$snapname};
3236         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3237         foreach_drive($snap, sub {
3238             my ($ds, $drive) = @_;
3239             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3240         });
3241     }
3242
3243     foreach my $volid (keys %$volhash) {
3244         &$func($volid, $volhash->{$volid}, @param);
3245     }
3246 }
3247
3248 sub conf_has_serial {
3249     my ($conf) = @_;
3250
3251     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3252         if ($conf->{"serial$i"}) {
3253             return 1;
3254         }
3255     }
3256
3257     return 0;
3258 }
3259
3260 sub vga_conf_has_spice {
3261     my ($vga) = @_;
3262
3263     my $vgaconf = parse_vga($vga);
3264     my $vgatype = $vgaconf->{type};
3265     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3266
3267     return $1 || 1;
3268 }
3269
3270 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3271 sub get_host_arch() {
3272     $host_arch = (POSIX::uname())[4] if !$host_arch;
3273     return $host_arch;
3274 }
3275
3276 sub is_native($) {
3277     my ($arch) = @_;
3278     return get_host_arch() eq $arch;
3279 }
3280
3281 my $default_machines = {
3282     x86_64 => 'pc',
3283     aarch64 => 'virt',
3284 };
3285
3286 sub get_basic_machine_info {
3287     my ($conf, $forcemachine) = @_;
3288
3289     my $arch = $conf->{arch} // get_host_arch();
3290     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3291     return ($arch, $machine);
3292 }
3293
3294 sub get_ovmf_files($) {
3295     my ($arch) = @_;
3296
3297     my $ovmf = $OVMF->{$arch}
3298         or die "no OVMF images known for architecture '$arch'\n";
3299
3300     return @$ovmf;
3301 }
3302
3303 my $Arch2Qemu = {
3304     aarch64 => '/usr/bin/qemu-system-aarch64',
3305     x86_64 => '/usr/bin/qemu-system-x86_64',
3306 };
3307 sub get_command_for_arch($) {
3308     my ($arch) = @_;
3309     return '/usr/bin/kvm' if is_native($arch);
3310
3311     my $cmd = $Arch2Qemu->{$arch}
3312         or die "don't know how to emulate architecture '$arch'\n";
3313     return $cmd;
3314 }
3315
3316 sub get_cpu_options {
3317     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3318
3319     my $cpuFlags = [];
3320     my $ostype = $conf->{ostype};
3321
3322     my $cpu = $kvm ? "kvm64" : "qemu64";
3323     if ($arch eq 'aarch64') {
3324         $cpu = 'cortex-a57';
3325     }
3326     if (my $cputype = $conf->{cpu}) {
3327         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3328             or die "Cannot parse cpu description: $cputype\n";
3329         $cpu = $cpuconf->{cputype};
3330         $kvm_off = 1 if $cpuconf->{hidden};
3331
3332         if (defined(my $flags = $cpuconf->{flags})) {
3333             push @$cpuFlags, split(";", $flags);
3334         }
3335     }
3336
3337     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3338
3339     push @$cpuFlags , '-x2apic'
3340         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3341
3342     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3343
3344     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3345
3346     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3347
3348         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3349         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3350     }
3351
3352     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3353
3354     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3355
3356     push @$cpuFlags, 'kvm=off' if $kvm_off;
3357
3358     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3359         push @$cpuFlags, "vendor=${cpu_vendor}"
3360             if $cpu_vendor ne 'default';
3361     } elsif ($arch ne 'aarch64') {
3362         die "internal error"; # should not happen
3363     }
3364
3365     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3366
3367     return ('-cpu', $cpu);
3368 }
3369
3370 sub config_to_command {
3371     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3372
3373     my $cmd = [];
3374     my $globalFlags = [];
3375     my $machineFlags = [];
3376     my $rtcFlags = [];
3377     my $devices = [];
3378     my $pciaddr = '';
3379     my $bridges = {};
3380     my $kvmver = kvm_user_version();
3381     my $vernum = 0; # unknown
3382     my $ostype = $conf->{ostype};
3383     my $winversion = windows_version($ostype);
3384     my $kvm = $conf->{kvm};
3385
3386     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3387     $kvm //= 1 if is_native($arch);
3388
3389     if ($kvm) {
3390         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3391             if !defined kvm_version();
3392     }
3393
3394     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3395         $vernum = $1*1000000+$2*1000;
3396     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3397         $vernum = $1*1000000+$2*1000+$3;
3398     }
3399
3400     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3401
3402     my $have_ovz = -f '/proc/vz/vestat';
3403
3404     my $q35 = machine_type_is_q35($conf);
3405     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3406     my $use_old_bios_files = undef;
3407     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3408
3409     my $cpuunits = defined($conf->{cpuunits}) ?
3410             $conf->{cpuunits} : $defaults->{cpuunits};
3411
3412     push @$cmd, get_command_for_arch($arch);
3413
3414     push @$cmd, '-id', $vmid;
3415
3416     my $vmname = $conf->{name} || "vm$vmid";
3417
3418     push @$cmd, '-name', $vmname;
3419
3420     my $use_virtio = 0;
3421
3422     my $qmpsocket = qmp_socket($vmid);
3423     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3424     push @$cmd, '-mon', "chardev=qmp,mode=control";
3425
3426     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3427         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3428         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3429     }
3430
3431     push @$cmd, '-pidfile' , pidfile_name($vmid);
3432
3433     push @$cmd, '-daemonize';
3434
3435     if ($conf->{smbios1}) {
3436         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3437     }
3438
3439     if ($conf->{vmgenid}) {
3440         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3441     }
3442
3443     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3444     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3445         die "uefi base image not found\n" if ! -f $ovmf_code;
3446
3447         my $path;
3448         my $format;
3449         if (my $efidisk = $conf->{efidisk0}) {
3450             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3451             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3452             $format = $d->{format};
3453             if ($storeid) {
3454                 $path = PVE::Storage::path($storecfg, $d->{file});
3455                 if (!defined($format)) {
3456                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3457                     $format = qemu_img_format($scfg, $volname);
3458                 }
3459             } else {
3460                 $path = $d->{file};
3461                 die "efidisk format must be specified\n"
3462                     if !defined($format);
3463             }
3464         } else {
3465             warn "no efidisk configured! Using temporary efivars disk.\n";
3466             $path = "/tmp/$vmid-ovmf.fd";
3467             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3468             $format = 'raw';
3469         }
3470
3471         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3472         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3473     }
3474
3475
3476     # add usb controllers
3477     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3478     push @$devices, @usbcontrollers if @usbcontrollers;
3479     my $vga = parse_vga($conf->{vga});
3480
3481     my $qxlnum = vga_conf_has_spice($conf->{vga});
3482     $vga->{type} = 'qxl' if $qxlnum;
3483
3484     if (!$vga->{type}) {
3485         if ($arch eq 'aarch64') {
3486             $vga->{type} = 'virtio';
3487         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3488             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3489         } else {
3490             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3491         }
3492     }
3493
3494     # enable absolute mouse coordinates (needed by vnc)
3495     my $tablet;
3496     if (defined($conf->{tablet})) {
3497         $tablet = $conf->{tablet};
3498     } else {
3499         $tablet = $defaults->{tablet};
3500         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3501         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3502     }
3503
3504     if ($tablet) {
3505         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3506         my $kbd = print_keyboarddevice_full($conf, $arch);
3507         push @$devices, '-device', $kbd if defined($kbd);
3508     }
3509
3510     my $kvm_off = 0;
3511     my $gpu_passthrough;
3512
3513     # host pci devices
3514     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3515         my $d = parse_hostpci($conf->{"hostpci$i"});
3516         next if !$d;
3517
3518         my $pcie = $d->{pcie};
3519         if($pcie){
3520             die "q35 machine model is not enabled" if !$q35;
3521             $pciaddr = print_pcie_addr("hostpci$i");
3522         }else{
3523             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3524         }
3525
3526         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3527         my $romfile = $d->{romfile};
3528
3529         my $xvga = '';
3530         if ($d->{'x-vga'}) {
3531             $xvga = ',x-vga=on';
3532             $kvm_off = 1;
3533             $vga->{type} = 'none';
3534             $gpu_passthrough = 1;
3535
3536             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3537                 $xvga = "";
3538             }
3539         }
3540         my $pcidevices = $d->{pciid};
3541         my $multifunction = 1 if @$pcidevices > 1;
3542
3543         my $j=0;
3544         foreach my $pcidevice (@$pcidevices) {
3545
3546             my $id = "hostpci$i";
3547             $id .= ".$j" if $multifunction;
3548             my $addr = $pciaddr;
3549             $addr .= ".$j" if $multifunction;
3550             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3551
3552             if($j == 0){
3553                 $devicestr .= "$rombar$xvga";
3554                 $devicestr .= ",multifunction=on" if $multifunction;
3555                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3556             }
3557
3558             push @$devices, '-device', $devicestr;
3559             $j++;
3560         }
3561     }
3562
3563     # usb devices
3564     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3565     push @$devices, @usbdevices if @usbdevices;
3566     # serial devices
3567     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3568         if (my $path = $conf->{"serial$i"}) {
3569             if ($path eq 'socket') {
3570                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3571                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3572                 # On aarch64, serial0 is the UART device. Qemu only allows
3573                 # connecting UART devices via the '-serial' command line, as
3574                 # the device has a fixed slot on the hardware...
3575                 if ($arch eq 'aarch64' && $i == 0) {
3576                     push @$devices, '-serial', "chardev:serial$i";
3577                 } else {
3578                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3579                 }
3580             } else {
3581                 die "no such serial device\n" if ! -c $path;
3582                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3583                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3584             }
3585         }
3586     }
3587
3588     # parallel devices
3589     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3590         if (my $path = $conf->{"parallel$i"}) {
3591             die "no such parallel device\n" if ! -c $path;
3592             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3593             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3594             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3595         }
3596     }
3597
3598
3599     my $sockets = 1;
3600     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3601     $sockets = $conf->{sockets} if  $conf->{sockets};
3602
3603     my $cores = $conf->{cores} || 1;
3604
3605     my $maxcpus = $sockets * $cores;
3606
3607     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3608
3609     my $allowed_vcpus = $cpuinfo->{cpus};
3610
3611     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3612         if ($allowed_vcpus < $maxcpus);
3613
3614     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3615
3616         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3617         for (my $i = 2; $i <= $vcpus; $i++)  {
3618             my $cpustr = print_cpu_device($conf,$i);
3619             push @$cmd, '-device', $cpustr;
3620         }
3621
3622     } else {
3623
3624         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3625     }
3626     push @$cmd, '-nodefaults';
3627
3628     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3629
3630     my $bootindex_hash = {};
3631     my $i = 1;
3632     foreach my $o (split(//, $bootorder)) {
3633         $bootindex_hash->{$o} = $i*100;
3634         $i++;
3635     }
3636
3637     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3638
3639     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3640
3641     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3642
3643     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3644         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3645         my $socket = vnc_socket($vmid);
3646         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3647     } else {
3648         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3649         push @$cmd, '-nographic';
3650     }
3651
3652     # time drift fix
3653     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3654
3655     my $useLocaltime = $conf->{localtime};
3656
3657     if ($winversion >= 5) { # windows
3658         $useLocaltime = 1 if !defined($conf->{localtime});
3659
3660         # use time drift fix when acpi is enabled
3661         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3662             $tdf = 1 if !defined($conf->{tdf});
3663         }
3664     }
3665
3666     if ($winversion >= 6) {
3667         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3668         push @$cmd, '-no-hpet';
3669     }
3670
3671     push @$rtcFlags, 'driftfix=slew' if $tdf;
3672
3673     if (!$kvm) {
3674         push @$machineFlags, 'accel=tcg';
3675     }
3676
3677     if ($machine_type) {
3678         push @$machineFlags, "type=${machine_type}";
3679     }
3680
3681     if ($conf->{startdate}) {
3682         push @$rtcFlags, "base=$conf->{startdate}";
3683     } elsif ($useLocaltime) {
3684         push @$rtcFlags, 'base=localtime';
3685     }
3686
3687     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3688
3689     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3690
3691     push @$cmd, '-S' if $conf->{freeze};
3692
3693     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3694
3695     # enable sound
3696     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3697     #push @$cmd, '-soundhw', 'es1370';
3698     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3699
3700     if (parse_guest_agent($conf)->{enabled}) {
3701         my $qgasocket = qmp_socket($vmid, 1);
3702         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3703         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3704         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3705         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3706     }
3707
3708     my $spice_port;
3709
3710     if ($qxlnum) {
3711         if ($qxlnum > 1) {
3712             if ($winversion){
3713                 for(my $i = 1; $i < $qxlnum; $i++){
3714                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3715                 }
3716             } else {
3717                 # assume other OS works like Linux
3718                 my ($ram, $vram) = ("134217728", "67108864");
3719                 if ($vga->{memory}) {
3720                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3721                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3722                 }
3723                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3724                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3725             }
3726         }
3727
3728         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3729
3730         my $nodename = PVE::INotify::nodename();
3731         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3732         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3733         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3734         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3735         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3736
3737         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3738
3739         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3740         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3741         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3742     }
3743
3744     # enable balloon by default, unless explicitly disabled
3745     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3746         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3747         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3748     }
3749
3750     if ($conf->{watchdog}) {
3751         my $wdopts = parse_watchdog($conf->{watchdog});
3752         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3753         my $watchdog = $wdopts->{model} || 'i6300esb';
3754         push @$devices, '-device', "$watchdog$pciaddr";
3755         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3756     }
3757
3758     my $vollist = [];
3759     my $scsicontroller = {};
3760     my $ahcicontroller = {};
3761     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3762
3763     # Add iscsi initiator name if available
3764     if (my $initiator = get_initiator_name()) {
3765         push @$devices, '-iscsi', "initiator-name=$initiator";
3766     }
3767
3768     foreach_drive($conf, sub {
3769         my ($ds, $drive) = @_;
3770
3771         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3772             push @$vollist, $drive->{file};
3773         }
3774
3775         # ignore efidisk here, already added in bios/fw handling code above
3776         return if $drive->{interface} eq 'efidisk';
3777
3778         $use_virtio = 1 if $ds =~ m/^virtio/;
3779
3780         if (drive_is_cdrom ($drive)) {
3781             if ($bootindex_hash->{d}) {
3782                 $drive->{bootindex} = $bootindex_hash->{d};
3783                 $bootindex_hash->{d} += 1;
3784             }
3785         } else {
3786             if ($bootindex_hash->{c}) {
3787                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3788                 $bootindex_hash->{c} += 1;
3789             }
3790         }
3791
3792         if($drive->{interface} eq 'virtio'){
3793            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3794         }
3795
3796         if ($drive->{interface} eq 'scsi') {
3797
3798             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3799
3800             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3801             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3802
3803             my $iothread = '';
3804             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3805                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3806                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3807             } elsif ($drive->{iothread}) {
3808                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3809             }
3810
3811             my $queues = '';
3812             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3813                 $queues = ",num_queues=$drive->{queues}";
3814             }
3815
3816             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3817             $scsicontroller->{$controller}=1;
3818         }
3819
3820         if ($drive->{interface} eq 'sata') {
3821            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3822            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3823            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3824            $ahcicontroller->{$controller}=1;
3825         }
3826
3827         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3828         push @$devices, '-drive',$drive_cmd;
3829         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3830     });
3831
3832     for (my $i = 0; $i < $MAX_NETS; $i++) {
3833          next if !$conf->{"net$i"};
3834          my $d = parse_net($conf->{"net$i"});
3835          next if !$d;
3836
3837          $use_virtio = 1 if $d->{model} eq 'virtio';
3838
3839          if ($bootindex_hash->{n}) {
3840             $d->{bootindex} = $bootindex_hash->{n};
3841             $bootindex_hash->{n} += 1;
3842          }
3843
3844          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3845          push @$devices, '-netdev', $netdevfull;
3846
3847          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3848          push @$devices, '-device', $netdevicefull;
3849     }
3850
3851     if (!$q35) {
3852         # add pci bridges
3853         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3854            $bridges->{1} = 1;
3855            $bridges->{2} = 1;
3856         }
3857
3858         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3859
3860         while (my ($k, $v) = each %$bridges) {
3861             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3862             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3863         }
3864     }
3865
3866     # add custom args
3867     if ($conf->{args}) {
3868         my $aa = PVE::Tools::split_args($conf->{args});
3869         push @$cmd, @$aa;
3870     }
3871
3872     push @$cmd, @$devices;
3873     push @$cmd, '-rtc', join(',', @$rtcFlags)
3874         if scalar(@$rtcFlags);
3875     push @$cmd, '-machine', join(',', @$machineFlags)
3876         if scalar(@$machineFlags);
3877     push @$cmd, '-global', join(',', @$globalFlags)
3878         if scalar(@$globalFlags);
3879
3880     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3881 }
3882
3883 sub vnc_socket {
3884     my ($vmid) = @_;
3885     return "${var_run_tmpdir}/$vmid.vnc";
3886 }
3887
3888 sub spice_port {
3889     my ($vmid) = @_;
3890
3891     my $res = vm_mon_cmd($vmid, 'query-spice');
3892
3893     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3894 }
3895
3896 sub qmp_socket {
3897     my ($vmid, $qga, $name) = @_;
3898     my $sockettype = $qga ? 'qga' : 'qmp';
3899     my $ext = $name ? '-'.$name : '';
3900     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3901 }
3902
3903 sub pidfile_name {
3904     my ($vmid) = @_;
3905     return "${var_run_tmpdir}/$vmid.pid";
3906 }
3907
3908 sub vm_devices_list {
3909     my ($vmid) = @_;
3910
3911     my $res = vm_mon_cmd($vmid, 'query-pci');
3912     my $devices_to_check = [];
3913     my $devices = {};
3914     foreach my $pcibus (@$res) {
3915         push @$devices_to_check, @{$pcibus->{devices}},
3916     }
3917
3918     while (@$devices_to_check) {
3919         my $to_check = [];
3920         for my $d (@$devices_to_check) {
3921             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3922             next if !$d->{'pci_bridge'};
3923
3924             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3925             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3926         }
3927         $devices_to_check = $to_check;
3928     }
3929
3930     my $resblock = vm_mon_cmd($vmid, 'query-block');
3931     foreach my $block (@$resblock) {
3932         if($block->{device} =~ m/^drive-(\S+)/){
3933                 $devices->{$1} = 1;
3934         }
3935     }
3936
3937     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3938     foreach my $mice (@$resmice) {
3939         if ($mice->{name} eq 'QEMU HID Tablet') {
3940             $devices->{tablet} = 1;
3941             last;
3942         }
3943     }
3944
3945     # for usb devices there is no query-usb
3946     # but we can iterate over the entries in
3947     # qom-list path=/machine/peripheral
3948     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3949     foreach my $per (@$resperipheral) {
3950         if ($per->{name} =~ m/^usb\d+$/) {
3951             $devices->{$per->{name}} = 1;
3952         }
3953     }
3954
3955     return $devices;
3956 }
3957
3958 sub vm_deviceplug {
3959     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3960
3961     my $q35 = machine_type_is_q35($conf);
3962
3963     my $devices_list = vm_devices_list($vmid);
3964     return 1 if defined($devices_list->{$deviceid});
3965
3966     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3967
3968     if ($deviceid eq 'tablet') {
3969
3970         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3971
3972     } elsif ($deviceid eq 'keyboard') {
3973
3974         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3975
3976     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3977
3978         die "usb hotplug currently not reliable\n";
3979         # since we can't reliably hot unplug all added usb devices
3980         # and usb passthrough disables live migration
3981         # we disable usb hotplugging for now
3982         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3983
3984     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3985
3986         qemu_iothread_add($vmid, $deviceid, $device);
3987
3988         qemu_driveadd($storecfg, $vmid, $device);
3989         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3990
3991         qemu_deviceadd($vmid, $devicefull);
3992         eval { qemu_deviceaddverify($vmid, $deviceid); };
3993         if (my $err = $@) {
3994             eval { qemu_drivedel($vmid, $deviceid); };
3995             warn $@ if $@;
3996             die $err;
3997         }
3998
3999     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4000
4001
4002         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4003         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4004         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4005
4006         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4007
4008         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4009             qemu_iothread_add($vmid, $deviceid, $device);
4010             $devicefull .= ",iothread=iothread-$deviceid";
4011         }
4012
4013         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4014             $devicefull .= ",num_queues=$device->{queues}";
4015         }
4016
4017         qemu_deviceadd($vmid, $devicefull);
4018         qemu_deviceaddverify($vmid, $deviceid);
4019
4020     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4021
4022         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4023         qemu_driveadd($storecfg, $vmid, $device);
4024
4025         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4026         eval { qemu_deviceadd($vmid, $devicefull); };
4027         if (my $err = $@) {
4028             eval { qemu_drivedel($vmid, $deviceid); };
4029             warn $@ if $@;
4030             die $err;
4031         }
4032
4033     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4034
4035         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4036
4037         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4038         my $use_old_bios_files = undef;
4039         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4040
4041         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4042         qemu_deviceadd($vmid, $netdevicefull);
4043         eval { qemu_deviceaddverify($vmid, $deviceid); };
4044         if (my $err = $@) {
4045             eval { qemu_netdevdel($vmid, $deviceid); };
4046             warn $@ if $@;
4047             die $err;
4048         }
4049
4050     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4051
4052         my $bridgeid = $2;
4053         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4054         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4055
4056         qemu_deviceadd($vmid, $devicefull);
4057         qemu_deviceaddverify($vmid, $deviceid);
4058
4059     } else {
4060         die "can't hotplug device '$deviceid'\n";
4061     }
4062
4063     return 1;
4064 }
4065
4066 # fixme: this should raise exceptions on error!
4067 sub vm_deviceunplug {
4068     my ($vmid, $conf, $deviceid) = @_;
4069
4070     my $devices_list = vm_devices_list($vmid);
4071     return 1 if !defined($devices_list->{$deviceid});
4072
4073     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4074
4075     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4076
4077         qemu_devicedel($vmid, $deviceid);
4078
4079     } elsif ($deviceid =~ m/^usb\d+$/) {
4080
4081         die "usb hotplug currently not reliable\n";
4082         # when unplugging usb devices this way,
4083         # there may be remaining usb controllers/hubs
4084         # so we disable it for now
4085         qemu_devicedel($vmid, $deviceid);
4086         qemu_devicedelverify($vmid, $deviceid);
4087
4088     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4089
4090         qemu_devicedel($vmid, $deviceid);
4091         qemu_devicedelverify($vmid, $deviceid);
4092         qemu_drivedel($vmid, $deviceid);
4093         qemu_iothread_del($conf, $vmid, $deviceid);
4094
4095     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4096
4097         qemu_devicedel($vmid, $deviceid);
4098         qemu_devicedelverify($vmid, $deviceid);
4099         qemu_iothread_del($conf, $vmid, $deviceid);
4100
4101     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4102
4103         qemu_devicedel($vmid, $deviceid);
4104         qemu_drivedel($vmid, $deviceid);
4105         qemu_deletescsihw($conf, $vmid, $deviceid);
4106
4107     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4108
4109         qemu_devicedel($vmid, $deviceid);
4110         qemu_devicedelverify($vmid, $deviceid);
4111         qemu_netdevdel($vmid, $deviceid);
4112
4113     } else {
4114         die "can't unplug device '$deviceid'\n";
4115     }
4116
4117     return 1;
4118 }
4119
4120 sub qemu_deviceadd {
4121     my ($vmid, $devicefull) = @_;
4122
4123     $devicefull = "driver=".$devicefull;
4124     my %options =  split(/[=,]/, $devicefull);
4125
4126     vm_mon_cmd($vmid, "device_add" , %options);
4127 }
4128
4129 sub qemu_devicedel {
4130     my ($vmid, $deviceid) = @_;
4131
4132     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4133 }
4134
4135 sub qemu_iothread_add {
4136     my($vmid, $deviceid, $device) = @_;
4137
4138     if ($device->{iothread}) {
4139         my $iothreads = vm_iothreads_list($vmid);
4140         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4141     }
4142 }
4143
4144 sub qemu_iothread_del {
4145     my($conf, $vmid, $deviceid) = @_;
4146
4147     my $device = parse_drive($deviceid, $conf->{$deviceid});
4148     if ($device->{iothread}) {
4149         my $iothreads = vm_iothreads_list($vmid);
4150         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4151     }
4152 }
4153
4154 sub qemu_objectadd {
4155     my($vmid, $objectid, $qomtype) = @_;
4156
4157     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4158
4159     return 1;
4160 }
4161
4162 sub qemu_objectdel {
4163     my($vmid, $objectid) = @_;
4164
4165     vm_mon_cmd($vmid, "object-del", id => $objectid);
4166
4167     return 1;
4168 }
4169
4170 sub qemu_driveadd {
4171     my ($storecfg, $vmid, $device) = @_;
4172
4173     my $drive = print_drive_full($storecfg, $vmid, $device);
4174     $drive =~ s/\\/\\\\/g;
4175     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4176
4177     # If the command succeeds qemu prints: "OK"
4178     return 1 if $ret =~ m/OK/s;
4179
4180     die "adding drive failed: $ret\n";
4181 }
4182
4183 sub qemu_drivedel {
4184     my($vmid, $deviceid) = @_;
4185
4186     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4187     $ret =~ s/^\s+//;
4188
4189     return 1 if $ret eq "";
4190
4191     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4192     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4193
4194     die "deleting drive $deviceid failed : $ret\n";
4195 }
4196
4197 sub qemu_deviceaddverify {
4198     my ($vmid, $deviceid) = @_;
4199
4200     for (my $i = 0; $i <= 5; $i++) {
4201          my $devices_list = vm_devices_list($vmid);
4202          return 1 if defined($devices_list->{$deviceid});
4203          sleep 1;
4204     }
4205
4206     die "error on hotplug device '$deviceid'\n";
4207 }
4208
4209
4210 sub qemu_devicedelverify {
4211     my ($vmid, $deviceid) = @_;
4212
4213     # need to verify that the device is correctly removed as device_del
4214     # is async and empty return is not reliable
4215
4216     for (my $i = 0; $i <= 5; $i++) {
4217          my $devices_list = vm_devices_list($vmid);
4218          return 1 if !defined($devices_list->{$deviceid});
4219          sleep 1;
4220     }
4221
4222     die "error on hot-unplugging device '$deviceid'\n";
4223 }
4224
4225 sub qemu_findorcreatescsihw {
4226     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4227
4228     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4229
4230     my $scsihwid="$controller_prefix$controller";
4231     my $devices_list = vm_devices_list($vmid);
4232
4233     if(!defined($devices_list->{$scsihwid})) {
4234         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4235     }
4236
4237     return 1;
4238 }
4239
4240 sub qemu_deletescsihw {
4241     my ($conf, $vmid, $opt) = @_;
4242
4243     my $device = parse_drive($opt, $conf->{$opt});
4244
4245     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4246         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4247         return 1;
4248     }
4249
4250     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4251
4252     my $devices_list = vm_devices_list($vmid);
4253     foreach my $opt (keys %{$devices_list}) {
4254         if (PVE::QemuServer::is_valid_drivename($opt)) {
4255             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4256             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4257                 return 1;
4258             }
4259         }
4260     }
4261
4262     my $scsihwid="scsihw$controller";
4263
4264     vm_deviceunplug($vmid, $conf, $scsihwid);
4265
4266     return 1;
4267 }
4268
4269 sub qemu_add_pci_bridge {
4270     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4271
4272     my $bridges = {};
4273
4274     my $bridgeid;
4275
4276     print_pci_addr($device, $bridges, $arch, $machine_type);
4277
4278     while (my ($k, $v) = each %$bridges) {
4279         $bridgeid = $k;
4280     }
4281     return 1 if !defined($bridgeid) || $bridgeid < 1;
4282
4283     my $bridge = "pci.$bridgeid";
4284     my $devices_list = vm_devices_list($vmid);
4285
4286     if (!defined($devices_list->{$bridge})) {
4287         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4288     }
4289
4290     return 1;
4291 }
4292
4293 sub qemu_set_link_status {
4294     my ($vmid, $device, $up) = @_;
4295
4296     vm_mon_cmd($vmid, "set_link", name => $device,
4297                up => $up ? JSON::true : JSON::false);
4298 }
4299
4300 sub qemu_netdevadd {
4301     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4302
4303     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4304     my %options =  split(/[=,]/, $netdev);
4305
4306     vm_mon_cmd($vmid, "netdev_add",  %options);
4307     return 1;
4308 }
4309
4310 sub qemu_netdevdel {
4311     my ($vmid, $deviceid) = @_;
4312
4313     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4314 }
4315
4316 sub qemu_usb_hotplug {
4317     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4318
4319     return if !$device;
4320
4321     # remove the old one first
4322     vm_deviceunplug($vmid, $conf, $deviceid);
4323
4324     # check if xhci controller is necessary and available
4325     if ($device->{usb3}) {
4326
4327         my $devicelist = vm_devices_list($vmid);
4328
4329         if (!$devicelist->{xhci}) {
4330             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4331             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4332         }
4333     }
4334     my $d = parse_usb_device($device->{host});
4335     $d->{usb3} = $device->{usb3};
4336
4337     # add the new one
4338     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4339 }
4340
4341 sub qemu_cpu_hotplug {
4342     my ($vmid, $conf, $vcpus) = @_;
4343
4344     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4345
4346     my $sockets = 1;
4347     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4348     $sockets = $conf->{sockets} if  $conf->{sockets};
4349     my $cores = $conf->{cores} || 1;
4350     my $maxcpus = $sockets * $cores;
4351
4352     $vcpus = $maxcpus if !$vcpus;
4353
4354     die "you can't add more vcpus than maxcpus\n"
4355         if $vcpus > $maxcpus;
4356
4357     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4358
4359     if ($vcpus < $currentvcpus) {
4360
4361         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4362
4363             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4364                 qemu_devicedel($vmid, "cpu$i");
4365                 my $retry = 0;
4366                 my $currentrunningvcpus = undef;
4367                 while (1) {
4368                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4369                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4370                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4371                     $retry++;
4372                     sleep 1;
4373                 }
4374                 #update conf after each succesfull cpu unplug
4375                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4376                 PVE::QemuConfig->write_config($vmid, $conf);
4377             }
4378         } else {
4379             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4380         }
4381
4382         return;
4383     }
4384
4385     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4386     die "vcpus in running vm does not match its configuration\n"
4387         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4388
4389     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4390
4391         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4392             my $cpustr = print_cpu_device($conf, $i);
4393             qemu_deviceadd($vmid, $cpustr);
4394
4395             my $retry = 0;
4396             my $currentrunningvcpus = undef;
4397             while (1) {
4398                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4399                 last if scalar(@{$currentrunningvcpus}) == $i;
4400                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4401                 sleep 1;
4402                 $retry++;
4403             }
4404             #update conf after each succesfull cpu hotplug
4405             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4406             PVE::QemuConfig->write_config($vmid, $conf);
4407         }
4408     } else {
4409
4410         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4411             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4412         }
4413     }
4414 }
4415
4416 sub qemu_block_set_io_throttle {
4417     my ($vmid, $deviceid,
4418         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4419         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4420         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4421         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4422
4423     return if !check_running($vmid) ;
4424
4425     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4426         bps => int($bps),
4427         bps_rd => int($bps_rd),
4428         bps_wr => int($bps_wr),
4429         iops => int($iops),
4430         iops_rd => int($iops_rd),
4431         iops_wr => int($iops_wr),
4432         bps_max => int($bps_max),
4433         bps_rd_max => int($bps_rd_max),
4434         bps_wr_max => int($bps_wr_max),
4435         iops_max => int($iops_max),
4436         iops_rd_max => int($iops_rd_max),
4437         iops_wr_max => int($iops_wr_max),
4438         bps_max_length => int($bps_max_length),
4439         bps_rd_max_length => int($bps_rd_max_length),
4440         bps_wr_max_length => int($bps_wr_max_length),
4441         iops_max_length => int($iops_max_length),
4442         iops_rd_max_length => int($iops_rd_max_length),
4443         iops_wr_max_length => int($iops_wr_max_length),
4444     );
4445
4446 }
4447
4448 # old code, only used to shutdown old VM after update
4449 sub __read_avail {
4450     my ($fh, $timeout) = @_;
4451
4452     my $sel = new IO::Select;
4453     $sel->add($fh);
4454
4455     my $res = '';
4456     my $buf;
4457
4458     my @ready;
4459     while (scalar (@ready = $sel->can_read($timeout))) {
4460         my $count;
4461         if ($count = $fh->sysread($buf, 8192)) {
4462             if ($buf =~ /^(.*)\(qemu\) $/s) {
4463                 $res .= $1;
4464                 last;
4465             } else {
4466                 $res .= $buf;
4467             }
4468         } else {
4469             if (!defined($count)) {
4470                 die "$!\n";
4471             }
4472             last;
4473         }
4474     }
4475
4476     die "monitor read timeout\n" if !scalar(@ready);
4477
4478     return $res;
4479 }
4480
4481 sub qemu_block_resize {
4482     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4483
4484     my $running = check_running($vmid);
4485
4486     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4487
4488     return if !$running;
4489
4490     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4491
4492 }
4493
4494 sub qemu_volume_snapshot {
4495     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4496
4497     my $running = check_running($vmid);
4498
4499     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4500         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4501     } else {
4502         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4503     }
4504 }
4505
4506 sub qemu_volume_snapshot_delete {
4507     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4508
4509     my $running = check_running($vmid);
4510
4511     if($running) {
4512
4513         $running = undef;
4514         my $conf = PVE::QemuConfig->load_config($vmid);
4515         foreach_drive($conf, sub {
4516             my ($ds, $drive) = @_;
4517             $running = 1 if $drive->{file} eq $volid;
4518         });
4519     }
4520
4521     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4522         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4523     } else {
4524         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4525     }
4526 }
4527
4528 sub set_migration_caps {
4529     my ($vmid) = @_;
4530
4531     my $cap_ref = [];
4532
4533     my $enabled_cap = {
4534         "auto-converge" => 1,
4535         "xbzrle" => 1,
4536         "x-rdma-pin-all" => 0,
4537         "zero-blocks" => 0,
4538         "compress" => 0
4539     };
4540
4541     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4542
4543     for my $supported_capability (@$supported_capabilities) {
4544         push @$cap_ref, {
4545             capability => $supported_capability->{capability},
4546             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4547         };
4548     }
4549
4550     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4551 }
4552
4553 my $fast_plug_option = {
4554     'lock' => 1,
4555     'name' => 1,
4556     'onboot' => 1,
4557     'shares' => 1,
4558     'startup' => 1,
4559     'description' => 1,
4560     'protection' => 1,
4561     'vmstatestorage' => 1,
4562 };
4563
4564 # hotplug changes in [PENDING]
4565 # $selection hash can be used to only apply specified options, for
4566 # example: { cores => 1 } (only apply changed 'cores')
4567 # $errors ref is used to return error messages
4568 sub vmconfig_hotplug_pending {
4569     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4570
4571     my $defaults = load_defaults();
4572     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4573
4574     # commit values which do not have any impact on running VM first
4575     # Note: those option cannot raise errors, we we do not care about
4576     # $selection and always apply them.
4577
4578     my $add_error = sub {
4579         my ($opt, $msg) = @_;
4580         $errors->{$opt} = "hotplug problem - $msg";
4581     };
4582
4583     my $changes = 0;
4584     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4585         if ($fast_plug_option->{$opt}) {
4586             $conf->{$opt} = $conf->{pending}->{$opt};
4587             delete $conf->{pending}->{$opt};
4588             $changes = 1;
4589         }
4590     }
4591
4592     if ($changes) {
4593         PVE::QemuConfig->write_config($vmid, $conf);
4594         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4595     }
4596
4597     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4598
4599     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4600     while (my ($opt, $force) = each %$pending_delete_hash) {
4601         next if $selection && !$selection->{$opt};
4602         eval {
4603             if ($opt eq 'hotplug') {
4604                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4605             } elsif ($opt eq 'tablet') {
4606                 die "skip\n" if !$hotplug_features->{usb};
4607                 if ($defaults->{tablet}) {
4608                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4609                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4610                         if $arch eq 'aarch64';
4611                 } else {
4612                     vm_deviceunplug($vmid, $conf, 'tablet');
4613                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4614                 }
4615             } elsif ($opt =~ m/^usb\d+/) {
4616                 die "skip\n";
4617                 # since we cannot reliably hot unplug usb devices
4618                 # we are disabling it
4619                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4620                 vm_deviceunplug($vmid, $conf, $opt);
4621             } elsif ($opt eq 'vcpus') {
4622                 die "skip\n" if !$hotplug_features->{cpu};
4623                 qemu_cpu_hotplug($vmid, $conf, undef);
4624             } elsif ($opt eq 'balloon') {
4625                 # enable balloon device is not hotpluggable
4626                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4627                 # here we reset the ballooning value to memory
4628                 my $balloon = $conf->{memory} || $defaults->{memory};
4629                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4630             } elsif ($fast_plug_option->{$opt}) {
4631                 # do nothing
4632             } elsif ($opt =~ m/^net(\d+)$/) {
4633                 die "skip\n" if !$hotplug_features->{network};
4634                 vm_deviceunplug($vmid, $conf, $opt);
4635             } elsif (is_valid_drivename($opt)) {
4636                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4637                 vm_deviceunplug($vmid, $conf, $opt);
4638                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4639             } elsif ($opt =~ m/^memory$/) {
4640                 die "skip\n" if !$hotplug_features->{memory};
4641                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4642             } elsif ($opt eq 'cpuunits') {
4643                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4644             } elsif ($opt eq 'cpulimit') {
4645                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4646             } else {
4647                 die "skip\n";
4648             }
4649         };
4650         if (my $err = $@) {
4651             &$add_error($opt, $err) if $err ne "skip\n";
4652         } else {
4653             # save new config if hotplug was successful
4654             delete $conf->{$opt};
4655             vmconfig_undelete_pending_option($conf, $opt);
4656             PVE::QemuConfig->write_config($vmid, $conf);
4657             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4658         }
4659     }
4660
4661     my $apply_pending_cloudinit;
4662     $apply_pending_cloudinit = sub {
4663         my ($key, $value) = @_;
4664         $apply_pending_cloudinit = sub {}; # once is enough
4665
4666         my @cloudinit_opts = keys %$confdesc_cloudinit;
4667         foreach my $opt (keys %{$conf->{pending}}) {
4668             next if !grep { $_ eq $opt } @cloudinit_opts;
4669             $conf->{$opt} = delete $conf->{pending}->{$opt};
4670         }
4671
4672         my $new_conf = { %$conf };
4673         $new_conf->{$key} = $value;
4674         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4675     };
4676
4677     foreach my $opt (keys %{$conf->{pending}}) {
4678         next if $selection && !$selection->{$opt};
4679         my $value = $conf->{pending}->{$opt};
4680         eval {
4681             if ($opt eq 'hotplug') {
4682                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4683             } elsif ($opt eq 'tablet') {
4684                 die "skip\n" if !$hotplug_features->{usb};
4685                 if ($value == 1) {
4686                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4687                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4688                         if $arch eq 'aarch64';
4689                 } elsif ($value == 0) {
4690                     vm_deviceunplug($vmid, $conf, 'tablet');
4691                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4692                 }
4693             } elsif ($opt =~ m/^usb\d+$/) {
4694                 die "skip\n";
4695                 # since we cannot reliably hot unplug usb devices
4696                 # we are disabling it
4697                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4698                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4699                 die "skip\n" if !$d;
4700                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4701             } elsif ($opt eq 'vcpus') {
4702                 die "skip\n" if !$hotplug_features->{cpu};
4703                 qemu_cpu_hotplug($vmid, $conf, $value);
4704             } elsif ($opt eq 'balloon') {
4705                 # enable/disable balloning device is not hotpluggable
4706                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4707                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4708                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4709
4710                 # allow manual ballooning if shares is set to zero
4711                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4712                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4713                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4714                 }
4715             } elsif ($opt =~ m/^net(\d+)$/) {
4716                 # some changes can be done without hotplug
4717                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4718                                     $vmid, $opt, $value, $arch, $machine_type);
4719             } elsif (is_valid_drivename($opt)) {
4720                 # some changes can be done without hotplug
4721                 my $drive = parse_drive($opt, $value);
4722                 if (drive_is_cloudinit($drive)) {
4723                     &$apply_pending_cloudinit($opt, $value);
4724                 }
4725                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4726                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4727             } elsif ($opt =~ m/^memory$/) { #dimms
4728                 die "skip\n" if !$hotplug_features->{memory};
4729                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4730             } elsif ($opt eq 'cpuunits') {
4731                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4732             } elsif ($opt eq 'cpulimit') {
4733                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4734                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4735             } else {
4736                 die "skip\n";  # skip non-hot-pluggable options
4737             }
4738         };
4739         if (my $err = $@) {
4740             &$add_error($opt, $err) if $err ne "skip\n";
4741         } else {
4742             # save new config if hotplug was successful
4743             $conf->{$opt} = $value;
4744             delete $conf->{pending}->{$opt};
4745             PVE::QemuConfig->write_config($vmid, $conf);
4746             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4747         }
4748     }
4749 }
4750
4751 sub try_deallocate_drive {
4752     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4753
4754     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4755         my $volid = $drive->{file};
4756         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4757             my $sid = PVE::Storage::parse_volume_id($volid);
4758             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4759
4760             # check if the disk is really unused
4761             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4762                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4763             PVE::Storage::vdisk_free($storecfg, $volid);
4764             return 1;
4765         } else {
4766             # If vm is not owner of this disk remove from config
4767             return 1;
4768         }
4769     }
4770
4771     return undef;
4772 }
4773
4774 sub vmconfig_delete_or_detach_drive {
4775     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4776
4777     my $drive = parse_drive($opt, $conf->{$opt});
4778
4779     my $rpcenv = PVE::RPCEnvironment::get();
4780     my $authuser = $rpcenv->get_user();
4781
4782     if ($force) {
4783         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4784         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4785     } else {
4786         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4787     }
4788 }
4789
4790 sub vmconfig_apply_pending {
4791     my ($vmid, $conf, $storecfg) = @_;
4792
4793     # cold plug
4794
4795     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4796     while (my ($opt, $force) = each %$pending_delete_hash) {
4797         die "internal error" if $opt =~ m/^unused/;
4798         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4799         if (!defined($conf->{$opt})) {
4800             vmconfig_undelete_pending_option($conf, $opt);
4801             PVE::QemuConfig->write_config($vmid, $conf);
4802         } elsif (is_valid_drivename($opt)) {
4803             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4804             vmconfig_undelete_pending_option($conf, $opt);
4805             delete $conf->{$opt};
4806             PVE::QemuConfig->write_config($vmid, $conf);
4807         } else {
4808             vmconfig_undelete_pending_option($conf, $opt);
4809             delete $conf->{$opt};
4810             PVE::QemuConfig->write_config($vmid, $conf);
4811         }
4812     }
4813
4814     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4815
4816     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4817         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4818
4819         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4820             # skip if nothing changed
4821         } elsif (is_valid_drivename($opt)) {
4822             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4823                 if defined($conf->{$opt});
4824             $conf->{$opt} = $conf->{pending}->{$opt};
4825         } else {
4826             $conf->{$opt} = $conf->{pending}->{$opt};
4827         }
4828
4829         delete $conf->{pending}->{$opt};
4830         PVE::QemuConfig->write_config($vmid, $conf);
4831     }
4832 }
4833
4834 my $safe_num_ne = sub {
4835     my ($a, $b) = @_;
4836
4837     return 0 if !defined($a) && !defined($b);
4838     return 1 if !defined($a);
4839     return 1 if !defined($b);
4840
4841     return $a != $b;
4842 };
4843
4844 my $safe_string_ne = sub {
4845     my ($a, $b) = @_;
4846
4847     return 0 if !defined($a) && !defined($b);
4848     return 1 if !defined($a);
4849     return 1 if !defined($b);
4850
4851     return $a ne $b;
4852 };
4853
4854 sub vmconfig_update_net {
4855     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4856
4857     my $newnet = parse_net($value);
4858
4859     if ($conf->{$opt}) {
4860         my $oldnet = parse_net($conf->{$opt});
4861
4862         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4863             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4864             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4865             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4866
4867             # for non online change, we try to hot-unplug
4868             die "skip\n" if !$hotplug;
4869             vm_deviceunplug($vmid, $conf, $opt);
4870         } else {
4871
4872             die "internal error" if $opt !~ m/net(\d+)/;
4873             my $iface = "tap${vmid}i$1";
4874
4875             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4876                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4877                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4878                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4879                 PVE::Network::tap_unplug($iface);
4880                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4881             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4882                 # Rate can be applied on its own but any change above needs to
4883                 # include the rate in tap_plug since OVS resets everything.
4884                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4885             }
4886
4887             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4888                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4889             }
4890
4891             return 1;
4892         }
4893     }
4894
4895     if ($hotplug) {
4896         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4897     } else {
4898         die "skip\n";
4899     }
4900 }
4901
4902 sub vmconfig_update_disk {
4903     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
4904
4905     # fixme: do we need force?
4906
4907     my $drive = parse_drive($opt, $value);
4908
4909     if ($conf->{$opt}) {
4910
4911         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4912
4913             my $media = $drive->{media} || 'disk';
4914             my $oldmedia = $old_drive->{media} || 'disk';
4915             die "unable to change media type\n" if $media ne $oldmedia;
4916
4917             if (!drive_is_cdrom($old_drive)) {
4918
4919                 if ($drive->{file} ne $old_drive->{file}) {
4920
4921                     die "skip\n" if !$hotplug;
4922
4923                     # unplug and register as unused
4924                     vm_deviceunplug($vmid, $conf, $opt);
4925                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4926
4927                 } else {
4928                     # update existing disk
4929
4930                     # skip non hotpluggable value
4931                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4932                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4933                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4934                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4935                         die "skip\n";
4936                     }
4937
4938                     # apply throttle
4939                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4940                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4941                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4942                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4943                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4944                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4945                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4946                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4947                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4948                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4949                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4950                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4951                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4952                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4953                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4954                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4955                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4956                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4957
4958                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4959                                                    ($drive->{mbps} || 0)*1024*1024,
4960                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4961                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4962                                                    $drive->{iops} || 0,
4963                                                    $drive->{iops_rd} || 0,
4964                                                    $drive->{iops_wr} || 0,
4965                                                    ($drive->{mbps_max} || 0)*1024*1024,
4966                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4967                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4968                                                    $drive->{iops_max} || 0,
4969                                                    $drive->{iops_rd_max} || 0,
4970                                                    $drive->{iops_wr_max} || 0,
4971                                                    $drive->{bps_max_length} || 1,
4972                                                    $drive->{bps_rd_max_length} || 1,
4973                                                    $drive->{bps_wr_max_length} || 1,
4974                                                    $drive->{iops_max_length} || 1,
4975                                                    $drive->{iops_rd_max_length} || 1,
4976                                                    $drive->{iops_wr_max_length} || 1);
4977
4978                     }
4979
4980                     return 1;
4981                 }
4982
4983             } else { # cdrom
4984
4985                 if ($drive->{file} eq 'none') {
4986                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4987                     if (drive_is_cloudinit($old_drive)) {
4988                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4989                     }
4990                 } else {
4991                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4992                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4993                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4994                 }
4995
4996                 return 1;
4997             }
4998         }
4999     }
5000
5001     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5002     # hotplug new disks
5003     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5004     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5005 }
5006
5007 sub vm_start {
5008     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5009         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5010
5011     PVE::QemuConfig->lock_config($vmid, sub {
5012         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5013
5014         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5015
5016         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5017
5018         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5019
5020         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5021             vmconfig_apply_pending($vmid, $conf, $storecfg);
5022             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5023         }
5024
5025         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5026
5027         my $defaults = load_defaults();
5028
5029         # set environment variable useful inside network script
5030         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5031
5032         my $local_volumes = {};
5033
5034         if ($targetstorage) {
5035             foreach_drive($conf, sub {
5036                 my ($ds, $drive) = @_;
5037
5038                 return if drive_is_cdrom($drive);
5039
5040                 my $volid = $drive->{file};
5041
5042                 return if !$volid;
5043
5044                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5045
5046                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5047                 return if $scfg->{shared};
5048                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5049             });
5050
5051             my $format = undef;
5052
5053             foreach my $opt (sort keys %$local_volumes) {
5054
5055                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5056                 my $drive = parse_drive($opt, $conf->{$opt});
5057
5058                 #if remote storage is specified, use default format
5059                 if ($targetstorage && $targetstorage ne "1") {
5060                     $storeid = $targetstorage;
5061                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5062                     $format = $defFormat;
5063                 } else {
5064                     #else we use same format than original
5065                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5066                     $format = qemu_img_format($scfg, $volid);
5067                 }
5068
5069                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5070                 my $newdrive = $drive;
5071                 $newdrive->{format} = $format;
5072                 $newdrive->{file} = $newvolid;
5073                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5074                 $local_volumes->{$opt} = $drivestr;
5075                 #pass drive to conf for command line
5076                 $conf->{$opt} = $drivestr;
5077             }
5078         }
5079
5080         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5081
5082         my $migrate_port = 0;
5083         my $migrate_uri;
5084         if ($statefile) {
5085             if ($statefile eq 'tcp') {
5086                 my $localip = "localhost";
5087                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5088                 my $nodename = PVE::INotify::nodename();
5089
5090                 if (!defined($migration_type)) {
5091                     if (defined($datacenterconf->{migration}->{type})) {
5092                         $migration_type = $datacenterconf->{migration}->{type};
5093                     } else {
5094                         $migration_type = 'secure';
5095                     }
5096                 }
5097
5098                 if ($migration_type eq 'insecure') {
5099                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5100                     if ($migrate_network_addr) {
5101                         $localip = $migrate_network_addr;
5102                     } else {
5103                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5104                     }
5105
5106                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5107                 }
5108
5109                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5110                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5111                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5112                 push @$cmd, '-incoming', $migrate_uri;
5113                 push @$cmd, '-S';
5114
5115             } elsif ($statefile eq 'unix') {
5116                 # should be default for secure migrations as a ssh TCP forward
5117                 # tunnel is not deterministic reliable ready and fails regurarly
5118                 # to set up in time, so use UNIX socket forwards
5119                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5120                 unlink $socket_addr;
5121
5122                 $migrate_uri = "unix:$socket_addr";
5123
5124                 push @$cmd, '-incoming', $migrate_uri;
5125                 push @$cmd, '-S';
5126
5127             } else {
5128                 push @$cmd, '-loadstate', $statefile;
5129             }
5130         } elsif ($paused) {
5131             push @$cmd, '-S';
5132         }
5133
5134         # host pci devices
5135         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5136           my $d = parse_hostpci($conf->{"hostpci$i"});
5137           next if !$d;
5138           my $pcidevices = $d->{pciid};
5139           foreach my $pcidevice (@$pcidevices) {
5140                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5141
5142                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5143                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5144                 die "no pci device info for device '$pciid'\n" if !$info;
5145                 die "can't unbind/bind pci group to vfio '$pciid'\n"
5146                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5147                 die "can't reset pci device '$pciid'\n"
5148                     if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5149           }
5150         }
5151
5152         PVE::Storage::activate_volumes($storecfg, $vollist);
5153
5154         if (!check_running($vmid, 1)) {
5155             eval {
5156                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5157                     outfunc => sub {}, errfunc => sub {});
5158             };
5159         }
5160
5161         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5162                                                   : $defaults->{cpuunits};
5163
5164         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5165         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5166
5167         my %properties = (
5168             Slice => 'qemu.slice',
5169             KillMode => 'none',
5170             CPUShares => $cpuunits
5171         );
5172
5173         if (my $cpulimit = $conf->{cpulimit}) {
5174             $properties{CPUQuota} = int($cpulimit * 100);
5175         }
5176         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5177
5178         my $run_qemu = sub {
5179             PVE::Tools::run_fork sub {
5180                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5181                 run_command($cmd, %run_params);
5182             };
5183         };
5184
5185         if ($conf->{hugepages}) {
5186
5187             my $code = sub {
5188                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5189                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5190
5191                 PVE::QemuServer::Memory::hugepages_mount();
5192                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5193
5194                 eval { $run_qemu->() };
5195                 if (my $err = $@) {
5196                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5197                     die $err;
5198                 }
5199
5200                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5201             };
5202             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5203
5204         } else {
5205             eval { $run_qemu->() };
5206         }
5207
5208         if (my $err = $@) {
5209             # deactivate volumes if start fails
5210             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5211             die "start failed: $err";
5212         }
5213
5214         print "migration listens on $migrate_uri\n" if $migrate_uri;
5215
5216         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5217             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5218             warn $@ if $@;
5219         }
5220
5221         #start nbd server for storage migration
5222         if ($targetstorage) {
5223             my $nodename = PVE::INotify::nodename();
5224             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5225             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5226             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5227             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5228
5229             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5230
5231             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5232
5233             foreach my $opt (sort keys %$local_volumes) {
5234                 my $volid = $local_volumes->{$opt};
5235                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5236                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5237                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5238             }
5239         }
5240
5241         if ($migratedfrom) {
5242             eval {
5243                 set_migration_caps($vmid);
5244             };
5245             warn $@ if $@;
5246
5247             if ($spice_port) {
5248                 print "spice listens on port $spice_port\n";
5249                 if ($spice_ticket) {
5250                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5251                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5252                 }
5253             }
5254
5255         } else {
5256             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5257                 if !$statefile && $conf->{balloon};
5258
5259             foreach my $opt (keys %$conf) {
5260                 next if $opt !~  m/^net\d+$/;
5261                 my $nicconf = parse_net($conf->{$opt});
5262                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5263             }
5264         }
5265
5266         vm_mon_cmd_nocheck($vmid, 'qom-set',
5267                     path => "machine/peripheral/balloon0",
5268                     property => "guest-stats-polling-interval",
5269                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5270
5271     });
5272 }
5273
5274 sub vm_mon_cmd {
5275     my ($vmid, $execute, %params) = @_;
5276
5277     my $cmd = { execute => $execute, arguments => \%params };
5278     vm_qmp_command($vmid, $cmd);
5279 }
5280
5281 sub vm_mon_cmd_nocheck {
5282     my ($vmid, $execute, %params) = @_;
5283
5284     my $cmd = { execute => $execute, arguments => \%params };
5285     vm_qmp_command($vmid, $cmd, 1);
5286 }
5287
5288 sub vm_qmp_command {
5289     my ($vmid, $cmd, $nocheck) = @_;
5290
5291     my $res;
5292
5293     my $timeout;
5294     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5295         $timeout = $cmd->{arguments}->{timeout};
5296         delete $cmd->{arguments}->{timeout};
5297     }
5298
5299     eval {
5300         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5301         my $sname = qmp_socket($vmid);
5302         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5303             my $qmpclient = PVE::QMPClient->new();
5304
5305             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5306         } else {
5307             die "unable to open monitor socket\n";
5308         }
5309     };
5310     if (my $err = $@) {
5311         syslog("err", "VM $vmid qmp command failed - $err");
5312         die $err;
5313     }
5314
5315     return $res;
5316 }
5317
5318 sub vm_human_monitor_command {
5319     my ($vmid, $cmdline) = @_;
5320
5321     my $res;
5322
5323     my $cmd = {
5324         execute => 'human-monitor-command',
5325         arguments => { 'command-line' => $cmdline},
5326     };
5327
5328     return vm_qmp_command($vmid, $cmd);
5329 }
5330
5331 sub vm_commandline {
5332     my ($storecfg, $vmid) = @_;
5333
5334     my $conf = PVE::QemuConfig->load_config($vmid);
5335
5336     my $defaults = load_defaults();
5337
5338     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5339
5340     return PVE::Tools::cmd2string($cmd);
5341 }
5342
5343 sub vm_reset {
5344     my ($vmid, $skiplock) = @_;
5345
5346     PVE::QemuConfig->lock_config($vmid, sub {
5347
5348         my $conf = PVE::QemuConfig->load_config($vmid);
5349
5350         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5351
5352         vm_mon_cmd($vmid, "system_reset");
5353     });
5354 }
5355
5356 sub get_vm_volumes {
5357     my ($conf) = @_;
5358
5359     my $vollist = [];
5360     foreach_volid($conf, sub {
5361         my ($volid, $attr) = @_;
5362
5363         return if $volid =~ m|^/|;
5364
5365         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5366         return if !$sid;
5367
5368         push @$vollist, $volid;
5369     });
5370
5371     return $vollist;
5372 }
5373
5374 sub vm_stop_cleanup {
5375     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5376
5377     eval {
5378
5379         if (!$keepActive) {
5380             my $vollist = get_vm_volumes($conf);
5381             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5382         }
5383
5384         foreach my $ext (qw(mon qmp pid vnc qga)) {
5385             unlink "/var/run/qemu-server/${vmid}.$ext";
5386         }
5387
5388         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5389     };
5390     warn $@ if $@; # avoid errors - just warn
5391 }
5392
5393 # Note: use $nockeck to skip tests if VM configuration file exists.
5394 # We need that when migration VMs to other nodes (files already moved)
5395 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5396 sub vm_stop {
5397     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5398
5399     $force = 1 if !defined($force) && !$shutdown;
5400
5401     if ($migratedfrom){
5402         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5403         kill 15, $pid if $pid;
5404         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5405         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5406         return;
5407     }
5408
5409     PVE::QemuConfig->lock_config($vmid, sub {
5410
5411         my $pid = check_running($vmid, $nocheck);
5412         return if !$pid;
5413
5414         my $conf;
5415         if (!$nocheck) {
5416             $conf = PVE::QemuConfig->load_config($vmid);
5417             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5418             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5419                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5420                 $timeout = $opts->{down} if $opts->{down};
5421             }
5422         }
5423
5424         $timeout = 60 if !defined($timeout);
5425
5426         eval {
5427             if ($shutdown) {
5428                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5429                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5430                 } else {
5431                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5432                 }
5433             } else {
5434                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5435             }
5436         };
5437         my $err = $@;
5438
5439         if (!$err) {
5440             my $count = 0;
5441             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5442                 $count++;
5443                 sleep 1;
5444             }
5445
5446             if ($count >= $timeout) {
5447                 if ($force) {
5448                     warn "VM still running - terminating now with SIGTERM\n";
5449                     kill 15, $pid;
5450                 } else {
5451                     die "VM quit/powerdown failed - got timeout\n";
5452                 }
5453             } else {
5454                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5455                 return;
5456             }
5457         } else {
5458             if ($force) {
5459                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5460                 kill 15, $pid;
5461             } else {
5462                 die "VM quit/powerdown failed\n";
5463             }
5464         }
5465
5466         # wait again
5467         $timeout = 10;
5468
5469         my $count = 0;
5470         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5471             $count++;
5472             sleep 1;
5473         }
5474
5475         if ($count >= $timeout) {
5476             warn "VM still running - terminating now with SIGKILL\n";
5477             kill 9, $pid;
5478             sleep 1;
5479         }
5480
5481         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5482    });
5483 }
5484
5485 sub vm_suspend {
5486     my ($vmid, $skiplock) = @_;
5487
5488     PVE::QemuConfig->lock_config($vmid, sub {
5489
5490         my $conf = PVE::QemuConfig->load_config($vmid);
5491
5492         PVE::QemuConfig->check_lock($conf)
5493             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5494
5495         vm_mon_cmd($vmid, "stop");
5496     });
5497 }
5498
5499 sub vm_resume {
5500     my ($vmid, $skiplock, $nocheck) = @_;
5501
5502     PVE::QemuConfig->lock_config($vmid, sub {
5503
5504         my $res = vm_mon_cmd($vmid, 'query-status');
5505         my $resume_cmd = 'cont';
5506
5507         if ($res->{status} && $res->{status} eq 'suspended') {
5508             $resume_cmd = 'system_wakeup';
5509         }
5510
5511         if (!$nocheck) {
5512
5513             my $conf = PVE::QemuConfig->load_config($vmid);
5514
5515             PVE::QemuConfig->check_lock($conf)
5516                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5517
5518             vm_mon_cmd($vmid, $resume_cmd);
5519
5520         } else {
5521             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5522         }
5523     });
5524 }
5525
5526 sub vm_sendkey {
5527     my ($vmid, $skiplock, $key) = @_;
5528
5529     PVE::QemuConfig->lock_config($vmid, sub {
5530
5531         my $conf = PVE::QemuConfig->load_config($vmid);
5532
5533         # there is no qmp command, so we use the human monitor command
5534         vm_human_monitor_command($vmid, "sendkey $key");
5535     });
5536 }
5537
5538 sub vm_destroy {
5539     my ($storecfg, $vmid, $skiplock) = @_;
5540
5541     PVE::QemuConfig->lock_config($vmid, sub {
5542
5543         my $conf = PVE::QemuConfig->load_config($vmid);
5544
5545         if (!check_running($vmid)) {
5546             destroy_vm($storecfg, $vmid, undef, $skiplock);
5547         } else {
5548             die "VM $vmid is running - destroy failed\n";
5549         }
5550     });
5551 }
5552
5553 # vzdump restore implementaion
5554
5555 sub tar_archive_read_firstfile {
5556     my $archive = shift;
5557
5558     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5559
5560     # try to detect archive type first
5561     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5562         die "unable to open file '$archive'\n";
5563     my $firstfile = <$fh>;
5564     kill 15, $pid;
5565     close $fh;
5566
5567     die "ERROR: archive contaions no data\n" if !$firstfile;
5568     chomp $firstfile;
5569
5570     return $firstfile;
5571 }
5572
5573 sub tar_restore_cleanup {
5574     my ($storecfg, $statfile) = @_;
5575
5576     print STDERR "starting cleanup\n";
5577
5578     if (my $fd = IO::File->new($statfile, "r")) {
5579         while (defined(my $line = <$fd>)) {
5580             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5581                 my $volid = $2;
5582                 eval {
5583                     if ($volid =~ m|^/|) {
5584                         unlink $volid || die 'unlink failed\n';
5585                     } else {
5586                         PVE::Storage::vdisk_free($storecfg, $volid);
5587                     }
5588                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5589                 };
5590                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5591             } else {
5592                 print STDERR "unable to parse line in statfile - $line";
5593             }
5594         }
5595         $fd->close();
5596     }
5597 }
5598
5599 sub restore_archive {
5600     my ($archive, $vmid, $user, $opts) = @_;
5601
5602     my $format = $opts->{format};
5603     my $comp;
5604
5605     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5606         $format = 'tar' if !$format;
5607         $comp = 'gzip';
5608     } elsif ($archive =~ m/\.tar$/) {
5609         $format = 'tar' if !$format;
5610     } elsif ($archive =~ m/.tar.lzo$/) {
5611         $format = 'tar' if !$format;
5612         $comp = 'lzop';
5613     } elsif ($archive =~ m/\.vma$/) {
5614         $format = 'vma' if !$format;
5615     } elsif ($archive =~ m/\.vma\.gz$/) {
5616         $format = 'vma' if !$format;
5617         $comp = 'gzip';
5618     } elsif ($archive =~ m/\.vma\.lzo$/) {
5619         $format = 'vma' if !$format;
5620         $comp = 'lzop';
5621     } else {
5622         $format = 'vma' if !$format; # default
5623     }
5624
5625     # try to detect archive format
5626     if ($format eq 'tar') {
5627         return restore_tar_archive($archive, $vmid, $user, $opts);
5628     } else {
5629         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5630     }
5631 }
5632
5633 sub restore_update_config_line {
5634     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5635
5636     return if $line =~ m/^\#qmdump\#/;
5637     return if $line =~ m/^\#vzdump\#/;
5638     return if $line =~ m/^lock:/;
5639     return if $line =~ m/^unused\d+:/;
5640     return if $line =~ m/^parent:/;
5641     return if $line =~ m/^template:/; # restored VM is never a template
5642
5643     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5644     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5645         # try to convert old 1.X settings
5646         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5647         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5648             my ($model, $macaddr) = split(/\=/, $devconfig);
5649             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5650             my $net = {
5651                 model => $model,
5652                 bridge => "vmbr$ind",
5653                 macaddr => $macaddr,
5654             };
5655             my $netstr = print_net($net);
5656
5657             print $outfd "net$cookie->{netcount}: $netstr\n";
5658             $cookie->{netcount}++;
5659         }
5660     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5661         my ($id, $netstr) = ($1, $2);
5662         my $net = parse_net($netstr);
5663         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5664         $netstr = print_net($net);
5665         print $outfd "$id: $netstr\n";
5666     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5667         my $virtdev = $1;
5668         my $value = $3;
5669         my $di = parse_drive($virtdev, $value);
5670         if (defined($di->{backup}) && !$di->{backup}) {
5671             print $outfd "#$line";
5672         } elsif ($map->{$virtdev}) {
5673             delete $di->{format}; # format can change on restore
5674             $di->{file} = $map->{$virtdev};
5675             $value = print_drive($vmid, $di);
5676             print $outfd "$virtdev: $value\n";
5677         } else {
5678             print $outfd $line;
5679         }
5680     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5681         my $vmgenid = $1;
5682         if ($vmgenid ne '0') {
5683             # always generate a new vmgenid if there was a valid one setup
5684             $vmgenid = generate_uuid();
5685         }
5686         print $outfd "vmgenid: $vmgenid\n";
5687     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5688         my ($uuid, $uuid_str);
5689         UUID::generate($uuid);
5690         UUID::unparse($uuid, $uuid_str);
5691         my $smbios1 = parse_smbios1($2);
5692         $smbios1->{uuid} = $uuid_str;
5693         print $outfd $1.print_smbios1($smbios1)."\n";
5694     } else {
5695         print $outfd $line;
5696     }
5697 }
5698
5699 sub scan_volids {
5700     my ($cfg, $vmid) = @_;
5701
5702     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5703
5704     my $volid_hash = {};
5705     foreach my $storeid (keys %$info) {
5706         foreach my $item (@{$info->{$storeid}}) {
5707             next if !($item->{volid} && $item->{size});
5708             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5709             $volid_hash->{$item->{volid}} = $item;
5710         }
5711     }
5712
5713     return $volid_hash;
5714 }
5715
5716 sub is_volume_in_use {
5717     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5718
5719     my $path = PVE::Storage::path($storecfg, $volid);
5720
5721     my $scan_config = sub {
5722         my ($cref, $snapname) = @_;
5723
5724         foreach my $key (keys %$cref) {
5725             my $value = $cref->{$key};
5726             if (is_valid_drivename($key)) {
5727                 next if $skip_drive && $key eq $skip_drive;
5728                 my $drive = parse_drive($key, $value);
5729                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5730                 return 1 if $volid eq $drive->{file};
5731                 if ($drive->{file} =~ m!^/!) {
5732                     return 1 if $drive->{file} eq $path;
5733                 } else {
5734                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5735                     next if !$storeid;
5736                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5737                     next if !$scfg;
5738                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5739                 }
5740             }
5741         }
5742
5743         return 0;
5744     };
5745
5746     return 1 if &$scan_config($conf);
5747
5748     undef $skip_drive;
5749
5750     foreach my $snapname (keys %{$conf->{snapshots}}) {
5751         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5752     }
5753
5754     return 0;
5755 }
5756
5757 sub update_disksize {
5758     my ($vmid, $conf, $volid_hash) = @_;
5759
5760     my $changes;
5761     my $prefix = "VM $vmid:";
5762
5763     # used and unused disks
5764     my $referenced = {};
5765
5766     # Note: it is allowed to define multiple storages with same path (alias), so
5767     # we need to check both 'volid' and real 'path' (two different volid can point
5768     # to the same path).
5769
5770     my $referencedpath = {};
5771
5772     # update size info
5773     foreach my $opt (keys %$conf) {
5774         if (is_valid_drivename($opt)) {
5775             my $drive = parse_drive($opt, $conf->{$opt});
5776             my $volid = $drive->{file};
5777             next if !$volid;
5778
5779             $referenced->{$volid} = 1;
5780             if ($volid_hash->{$volid} &&
5781                 (my $path = $volid_hash->{$volid}->{path})) {
5782                 $referencedpath->{$path} = 1;
5783             }
5784
5785             next if drive_is_cdrom($drive);
5786             next if !$volid_hash->{$volid};
5787
5788             $drive->{size} = $volid_hash->{$volid}->{size};
5789             my $new = print_drive($vmid, $drive);
5790             if ($new ne $conf->{$opt}) {
5791                 $changes = 1;
5792                 $conf->{$opt} = $new;
5793                 print "$prefix update disk '$opt' information.\n";
5794             }
5795         }
5796     }
5797
5798     # remove 'unusedX' entry if volume is used
5799     foreach my $opt (keys %$conf) {
5800         next if $opt !~ m/^unused\d+$/;
5801         my $volid = $conf->{$opt};
5802         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5803         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5804             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5805             $changes = 1;
5806             delete $conf->{$opt};
5807         }
5808
5809         $referenced->{$volid} = 1;
5810         $referencedpath->{$path} = 1 if $path;
5811     }
5812
5813     foreach my $volid (sort keys %$volid_hash) {
5814         next if $volid =~ m/vm-$vmid-state-/;
5815         next if $referenced->{$volid};
5816         my $path = $volid_hash->{$volid}->{path};
5817         next if !$path; # just to be sure
5818         next if $referencedpath->{$path};
5819         $changes = 1;
5820         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5821         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5822         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5823     }
5824
5825     return $changes;
5826 }
5827
5828 sub rescan {
5829     my ($vmid, $nolock, $dryrun) = @_;
5830
5831     my $cfg = PVE::Storage::config();
5832
5833     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5834     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5835     foreach my $stor (keys %{$cfg->{ids}}) {
5836         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5837     }
5838
5839     print "rescan volumes...\n";
5840     my $volid_hash = scan_volids($cfg, $vmid);
5841
5842     my $updatefn =  sub {
5843         my ($vmid) = @_;
5844
5845         my $conf = PVE::QemuConfig->load_config($vmid);
5846
5847         PVE::QemuConfig->check_lock($conf);
5848
5849         my $vm_volids = {};
5850         foreach my $volid (keys %$volid_hash) {
5851             my $info = $volid_hash->{$volid};
5852             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5853         }
5854
5855         my $changes = update_disksize($vmid, $conf, $vm_volids);
5856
5857         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5858     };
5859
5860     if (defined($vmid)) {
5861         if ($nolock) {
5862             &$updatefn($vmid);
5863         } else {
5864             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5865         }
5866     } else {
5867         my $vmlist = config_list();
5868         foreach my $vmid (keys %$vmlist) {
5869             if ($nolock) {
5870                 &$updatefn($vmid);
5871             } else {
5872                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5873             }
5874         }
5875     }
5876 }
5877
5878 sub restore_vma_archive {
5879     my ($archive, $vmid, $user, $opts, $comp) = @_;
5880
5881     my $readfrom = $archive;
5882
5883     my $cfg = PVE::Storage::config();
5884     my $commands = [];
5885     my $bwlimit = $opts->{bwlimit};
5886
5887     my $dbg_cmdstring = '';
5888     my $add_pipe = sub {
5889         my ($cmd) = @_;
5890         push @$commands, $cmd;
5891         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5892         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5893         $readfrom = '-';
5894     };
5895
5896     my $input = undef;
5897     if ($archive eq '-') {
5898         $input = '<&STDIN';
5899     } else {
5900         # If we use a backup from a PVE defined storage we also consider that
5901         # storage's rate limit:
5902         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5903         if (defined($volid)) {
5904             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5905             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5906             if ($readlimit) {
5907                 print STDERR "applying read rate limit: $readlimit\n";
5908                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5909                 $add_pipe->($cstream);
5910             }
5911         }
5912     }
5913
5914     if ($comp) {
5915         my $cmd;
5916         if ($comp eq 'gzip') {
5917             $cmd = ['zcat', $readfrom];
5918         } elsif ($comp eq 'lzop') {
5919             $cmd = ['lzop', '-d', '-c', $readfrom];
5920         } else {
5921             die "unknown compression method '$comp'\n";
5922         }
5923         $add_pipe->($cmd);
5924     }
5925
5926     my $tmpdir = "/var/tmp/vzdumptmp$$";
5927     rmtree $tmpdir;
5928
5929     # disable interrupts (always do cleanups)
5930     local $SIG{INT} =
5931         local $SIG{TERM} =
5932         local $SIG{QUIT} =
5933         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5934
5935     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5936     POSIX::mkfifo($mapfifo, 0600);
5937     my $fifofh;
5938
5939     my $openfifo = sub {
5940         open($fifofh, '>', $mapfifo) || die $!;
5941     };
5942
5943     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5944
5945     my $oldtimeout;
5946     my $timeout = 5;
5947
5948     my $devinfo = {};
5949
5950     my $rpcenv = PVE::RPCEnvironment::get();
5951
5952     my $conffile = PVE::QemuConfig->config_file($vmid);
5953     my $tmpfn = "$conffile.$$.tmp";
5954
5955     # Note: $oldconf is undef if VM does not exists
5956     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5957     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5958
5959     my %storage_limits;
5960
5961     my $print_devmap = sub {
5962         my $virtdev_hash = {};
5963
5964         my $cfgfn = "$tmpdir/qemu-server.conf";
5965
5966         # we can read the config - that is already extracted
5967         my $fh = IO::File->new($cfgfn, "r") ||
5968             "unable to read qemu-server.conf - $!\n";
5969
5970         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5971         if (-f $fwcfgfn) {
5972             my $pve_firewall_dir = '/etc/pve/firewall';
5973             mkdir $pve_firewall_dir; # make sure the dir exists
5974             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5975         }
5976
5977         while (defined(my $line = <$fh>)) {
5978             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5979                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5980                 die "archive does not contain data for drive '$virtdev'\n"
5981                     if !$devinfo->{$devname};
5982                 if (defined($opts->{storage})) {
5983                     $storeid = $opts->{storage} || 'local';
5984                 } elsif (!$storeid) {
5985                     $storeid = 'local';
5986                 }
5987                 $format = 'raw' if !$format;
5988                 $devinfo->{$devname}->{devname} = $devname;
5989                 $devinfo->{$devname}->{virtdev} = $virtdev;
5990                 $devinfo->{$devname}->{format} = $format;
5991                 $devinfo->{$devname}->{storeid} = $storeid;
5992
5993                 # check permission on storage
5994                 my $pool = $opts->{pool}; # todo: do we need that?
5995                 if ($user ne 'root@pam') {
5996                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5997                 }
5998
5999                 $storage_limits{$storeid} = $bwlimit;
6000
6001                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6002             }
6003         }
6004
6005         foreach my $key (keys %storage_limits) {
6006             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6007             next if !$limit;
6008             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6009             $storage_limits{$key} = $limit * 1024;
6010         }
6011
6012         foreach my $devname (keys %$devinfo) {
6013             die "found no device mapping information for device '$devname'\n"
6014                 if !$devinfo->{$devname}->{virtdev};
6015         }
6016
6017         # create empty/temp config
6018         if ($oldconf) {
6019             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6020             foreach_drive($oldconf, sub {
6021                 my ($ds, $drive) = @_;
6022
6023                 return if drive_is_cdrom($drive);
6024
6025                 my $volid = $drive->{file};
6026
6027                 return if !$volid || $volid =~ m|^/|;
6028
6029                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6030                 return if !$path || !$owner || ($owner != $vmid);
6031
6032                 # Note: only delete disk we want to restore
6033                 # other volumes will become unused
6034                 if ($virtdev_hash->{$ds}) {
6035                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6036                     if (my $err = $@) {
6037                         warn $err;
6038                     }
6039                 }
6040             });
6041
6042             # delete vmstate files
6043             # since after the restore we have no snapshots anymore
6044             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6045                 my $snap = $oldconf->{snapshots}->{$snapname};
6046                 if ($snap->{vmstate}) {
6047                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6048                     if (my $err = $@) {
6049                         warn $err;
6050                     }
6051                 }
6052             }
6053         }
6054
6055         my $map = {};
6056         foreach my $virtdev (sort keys %$virtdev_hash) {
6057             my $d = $virtdev_hash->{$virtdev};
6058             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6059             my $storeid = $d->{storeid};
6060             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6061
6062             my $map_opts = '';
6063             if (my $limit = $storage_limits{$storeid}) {
6064                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6065             }
6066
6067             # test if requested format is supported
6068             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6069             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6070             $d->{format} = $defFormat if !$supported;
6071
6072             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6073                                                   $d->{format}, undef, $alloc_size);
6074             print STDERR "new volume ID is '$volid'\n";
6075             $d->{volid} = $volid;
6076             my $path = PVE::Storage::path($cfg, $volid);
6077
6078             PVE::Storage::activate_volumes($cfg,[$volid]);
6079
6080             my $write_zeros = 1;
6081             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6082                 $write_zeros = 0;
6083             }
6084
6085             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6086
6087             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6088             $map->{$virtdev} = $volid;
6089         }
6090
6091         $fh->seek(0, 0) || die "seek failed - $!\n";
6092
6093         my $outfd = new IO::File ($tmpfn, "w") ||
6094             die "unable to write config for VM $vmid\n";
6095
6096         my $cookie = { netcount => 0 };
6097         while (defined(my $line = <$fh>)) {
6098             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6099         }
6100
6101         $fh->close();
6102         $outfd->close();
6103     };
6104
6105     eval {
6106         # enable interrupts
6107         local $SIG{INT} =
6108             local $SIG{TERM} =
6109             local $SIG{QUIT} =
6110             local $SIG{HUP} =
6111             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6112         local $SIG{ALRM} = sub { die "got timeout\n"; };
6113
6114         $oldtimeout = alarm($timeout);
6115
6116         my $parser = sub {
6117             my $line = shift;
6118
6119             print "$line\n";
6120
6121             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6122                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6123                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6124             } elsif ($line =~ m/^CTIME: /) {
6125                 # we correctly received the vma config, so we can disable
6126                 # the timeout now for disk allocation (set to 10 minutes, so
6127                 # that we always timeout if something goes wrong)
6128                 alarm(600);
6129                 &$print_devmap();
6130                 print $fifofh "done\n";
6131                 my $tmp = $oldtimeout || 0;
6132                 $oldtimeout = undef;
6133                 alarm($tmp);
6134                 close($fifofh);
6135             }
6136         };
6137
6138         print "restore vma archive: $dbg_cmdstring\n";
6139         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6140     };
6141     my $err = $@;
6142
6143     alarm($oldtimeout) if $oldtimeout;
6144
6145     my $vollist = [];
6146     foreach my $devname (keys %$devinfo) {
6147         my $volid = $devinfo->{$devname}->{volid};
6148         push @$vollist, $volid if $volid;
6149     }
6150
6151     PVE::Storage::deactivate_volumes($cfg, $vollist);
6152
6153     unlink $mapfifo;
6154
6155     if ($err) {
6156         rmtree $tmpdir;
6157         unlink $tmpfn;
6158
6159         foreach my $devname (keys %$devinfo) {
6160             my $volid = $devinfo->{$devname}->{volid};
6161             next if !$volid;
6162             eval {
6163                 if ($volid =~ m|^/|) {
6164                     unlink $volid || die 'unlink failed\n';
6165                 } else {
6166                     PVE::Storage::vdisk_free($cfg, $volid);
6167                 }
6168                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6169             };
6170             print STDERR "unable to cleanup '$volid' - $@" if $@;
6171         }
6172         die $err;
6173     }
6174
6175     rmtree $tmpdir;
6176
6177     rename($tmpfn, $conffile) ||
6178         die "unable to commit configuration file '$conffile'\n";
6179
6180     PVE::Cluster::cfs_update(); # make sure we read new file
6181
6182     eval { rescan($vmid, 1); };
6183     warn $@ if $@;
6184 }
6185
6186 sub restore_tar_archive {
6187     my ($archive, $vmid, $user, $opts) = @_;
6188
6189     if ($archive ne '-') {
6190         my $firstfile = tar_archive_read_firstfile($archive);
6191         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6192             if $firstfile ne 'qemu-server.conf';
6193     }
6194
6195     my $storecfg = PVE::Storage::config();
6196
6197     # destroy existing data - keep empty config
6198     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6199     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6200
6201     my $tocmd = "/usr/lib/qemu-server/qmextract";
6202
6203     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6204     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6205     $tocmd .= ' --prealloc' if $opts->{prealloc};
6206     $tocmd .= ' --info' if $opts->{info};
6207
6208     # tar option "xf" does not autodetect compression when read from STDIN,
6209     # so we pipe to zcat
6210     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6211         PVE::Tools::shellquote("--to-command=$tocmd");
6212
6213     my $tmpdir = "/var/tmp/vzdumptmp$$";
6214     mkpath $tmpdir;
6215
6216     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6217     local $ENV{VZDUMP_VMID} = $vmid;
6218     local $ENV{VZDUMP_USER} = $user;
6219
6220     my $conffile = PVE::QemuConfig->config_file($vmid);
6221     my $tmpfn = "$conffile.$$.tmp";
6222
6223     # disable interrupts (always do cleanups)
6224     local $SIG{INT} =
6225         local $SIG{TERM} =
6226         local $SIG{QUIT} =
6227         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6228
6229     eval {
6230         # enable interrupts
6231         local $SIG{INT} =
6232             local $SIG{TERM} =
6233             local $SIG{QUIT} =
6234             local $SIG{HUP} =
6235             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6236
6237         if ($archive eq '-') {
6238             print "extracting archive from STDIN\n";
6239             run_command($cmd, input => "<&STDIN");
6240         } else {
6241             print "extracting archive '$archive'\n";
6242             run_command($cmd);
6243         }
6244
6245         return if $opts->{info};
6246
6247         # read new mapping
6248         my $map = {};
6249         my $statfile = "$tmpdir/qmrestore.stat";
6250         if (my $fd = IO::File->new($statfile, "r")) {
6251             while (defined (my $line = <$fd>)) {
6252                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6253                     $map->{$1} = $2 if $1;
6254                 } else {
6255                     print STDERR "unable to parse line in statfile - $line\n";
6256                 }
6257             }
6258             $fd->close();
6259         }
6260
6261         my $confsrc = "$tmpdir/qemu-server.conf";
6262
6263         my $srcfd = new IO::File($confsrc, "r") ||
6264             die "unable to open file '$confsrc'\n";
6265
6266         my $outfd = new IO::File ($tmpfn, "w") ||
6267             die "unable to write config for VM $vmid\n";
6268
6269         my $cookie = { netcount => 0 };
6270         while (defined (my $line = <$srcfd>)) {
6271             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6272         }
6273
6274         $srcfd->close();
6275         $outfd->close();
6276     };
6277     my $err = $@;
6278
6279     if ($err) {
6280
6281         unlink $tmpfn;
6282
6283         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6284
6285         die $err;
6286     }
6287
6288     rmtree $tmpdir;
6289
6290     rename $tmpfn, $conffile ||
6291         die "unable to commit configuration file '$conffile'\n";
6292
6293     PVE::Cluster::cfs_update(); # make sure we read new file
6294
6295     eval { rescan($vmid, 1); };
6296     warn $@ if $@;
6297 };
6298
6299 sub foreach_storage_used_by_vm {
6300     my ($conf, $func) = @_;
6301
6302     my $sidhash = {};
6303
6304     foreach_drive($conf, sub {
6305         my ($ds, $drive) = @_;
6306         return if drive_is_cdrom($drive);
6307
6308         my $volid = $drive->{file};
6309
6310         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6311         $sidhash->{$sid} = $sid if $sid;
6312     });
6313
6314     foreach my $sid (sort keys %$sidhash) {
6315         &$func($sid);
6316     }
6317 }
6318
6319 sub do_snapshots_with_qemu {
6320     my ($storecfg, $volid) = @_;
6321
6322     my $storage_name = PVE::Storage::parse_volume_id($volid);
6323
6324     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6325         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6326         return 1;
6327     }
6328
6329     if ($volid =~ m/\.(qcow2|qed)$/){
6330         return 1;
6331     }
6332
6333     return undef;
6334 }
6335
6336 sub qga_check_running {
6337     my ($vmid, $nowarn) = @_;
6338
6339     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6340     if ($@) {
6341         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6342         return 0;
6343     }
6344     return 1;
6345 }
6346
6347 sub template_create {
6348     my ($vmid, $conf, $disk) = @_;
6349
6350     my $storecfg = PVE::Storage::config();
6351
6352     foreach_drive($conf, sub {
6353         my ($ds, $drive) = @_;
6354
6355         return if drive_is_cdrom($drive);
6356         return if $disk && $ds ne $disk;
6357
6358         my $volid = $drive->{file};
6359         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6360
6361         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6362         $drive->{file} = $voliddst;
6363         $conf->{$ds} = print_drive($vmid, $drive);
6364         PVE::QemuConfig->write_config($vmid, $conf);
6365     });
6366 }
6367
6368 sub qemu_img_convert {
6369     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6370
6371     my $storecfg = PVE::Storage::config();
6372     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6373     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6374
6375     if ($src_storeid && $dst_storeid) {
6376
6377         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6378
6379         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6380         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6381
6382         my $src_format = qemu_img_format($src_scfg, $src_volname);
6383         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6384
6385         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6386         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6387
6388         my $cmd = [];
6389         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6390         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6391         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6392         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6393         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6394         if ($is_zero_initialized) {
6395             push @$cmd, "zeroinit:$dst_path";
6396         } else {
6397             push @$cmd, $dst_path;
6398         }
6399
6400         my $parser = sub {
6401             my $line = shift;
6402             if($line =~ m/\((\S+)\/100\%\)/){
6403                 my $percent = $1;
6404                 my $transferred = int($size * $percent / 100);
6405                 my $remaining = $size - $transferred;
6406
6407                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6408             }
6409
6410         };
6411
6412         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6413         my $err = $@;
6414         die "copy failed: $err" if $err;
6415     }
6416 }
6417
6418 sub qemu_img_format {
6419     my ($scfg, $volname) = @_;
6420
6421     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6422         return $1;
6423     } else {
6424         return "raw";
6425     }
6426 }
6427
6428 sub qemu_drive_mirror {
6429     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6430
6431     $jobs = {} if !$jobs;
6432
6433     my $qemu_target;
6434     my $format;
6435     $jobs->{"drive-$drive"} = {};
6436
6437     if ($dst_volid =~ /^nbd:/) {
6438         $qemu_target = $dst_volid;
6439         $format = "nbd";
6440     } else {
6441         my $storecfg = PVE::Storage::config();
6442         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6443
6444         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6445
6446         $format = qemu_img_format($dst_scfg, $dst_volname);
6447
6448         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6449
6450         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6451     }
6452
6453     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6454     $opts->{format} = $format if $format;
6455
6456     print "drive mirror is starting for drive-$drive\n";
6457
6458     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6459
6460     if (my $err = $@) {
6461         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6462         die "mirroring error: $err";
6463     }
6464
6465     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6466 }
6467
6468 sub qemu_drive_mirror_monitor {
6469     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6470
6471     eval {
6472         my $err_complete = 0;
6473
6474         while (1) {
6475             die "storage migration timed out\n" if $err_complete > 300;
6476
6477             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6478
6479             my $running_mirror_jobs = {};
6480             foreach my $stat (@$stats) {
6481                 next if $stat->{type} ne 'mirror';
6482                 $running_mirror_jobs->{$stat->{device}} = $stat;
6483             }
6484
6485             my $readycounter = 0;
6486
6487             foreach my $job (keys %$jobs) {
6488
6489                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6490                     print "$job : finished\n";
6491                     delete $jobs->{$job};
6492                     next;
6493                 }
6494
6495                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6496
6497                 my $busy = $running_mirror_jobs->{$job}->{busy};
6498                 my $ready = $running_mirror_jobs->{$job}->{ready};
6499                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6500                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6501                     my $remaining = $total - $transferred;
6502                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6503
6504                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6505                 }
6506
6507                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6508             }
6509
6510             last if scalar(keys %$jobs) == 0;
6511
6512             if ($readycounter == scalar(keys %$jobs)) {
6513                 print "all mirroring jobs are ready \n";
6514                 last if $skipcomplete; #do the complete later
6515
6516                 if ($vmiddst && $vmiddst != $vmid) {
6517                     my $agent_running = $qga && qga_check_running($vmid);
6518                     if ($agent_running) {
6519                         print "freeze filesystem\n";
6520                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6521                     } else {
6522                         print "suspend vm\n";
6523                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6524                     }
6525
6526                     # if we clone a disk for a new target vm, we don't switch the disk
6527                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6528
6529                     if ($agent_running) {
6530                         print "unfreeze filesystem\n";
6531                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6532                     } else {
6533                         print "resume vm\n";
6534                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6535                     }
6536
6537                     last;
6538                 } else {
6539
6540                     foreach my $job (keys %$jobs) {
6541                         # try to switch the disk if source and destination are on the same guest
6542                         print "$job: Completing block job...\n";
6543
6544                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6545                         if ($@ =~ m/cannot be completed/) {
6546                             print "$job: Block job cannot be completed, try again.\n";
6547                             $err_complete++;
6548                         }else {
6549                             print "$job: Completed successfully.\n";
6550                             $jobs->{$job}->{complete} = 1;
6551                         }
6552                     }
6553                 }
6554             }
6555             sleep 1;
6556         }
6557     };
6558     my $err = $@;
6559
6560     if ($err) {
6561         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6562         die "mirroring error: $err";
6563     }
6564
6565 }
6566
6567 sub qemu_blockjobs_cancel {
6568     my ($vmid, $jobs) = @_;
6569
6570     foreach my $job (keys %$jobs) {
6571         print "$job: Cancelling block job\n";
6572         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6573         $jobs->{$job}->{cancel} = 1;
6574     }
6575
6576     while (1) {
6577         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6578
6579         my $running_jobs = {};
6580         foreach my $stat (@$stats) {
6581             $running_jobs->{$stat->{device}} = $stat;
6582         }
6583
6584         foreach my $job (keys %$jobs) {
6585
6586             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6587                 print "$job: Done.\n";
6588                 delete $jobs->{$job};
6589             }
6590         }
6591
6592         last if scalar(keys %$jobs) == 0;
6593
6594         sleep 1;
6595     }
6596 }
6597
6598 sub clone_disk {
6599     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6600         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6601
6602     my $newvolid;
6603
6604     if (!$full) {
6605         print "create linked clone of drive $drivename ($drive->{file})\n";
6606         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6607         push @$newvollist, $newvolid;
6608     } else {
6609
6610         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6611         $storeid = $storage if $storage;
6612
6613         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6614         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6615
6616         print "create full clone of drive $drivename ($drive->{file})\n";
6617         my $name = undef;
6618         if (drive_is_cloudinit($drive)) {
6619             $name = "vm-$newvmid-cloudinit";
6620             # cloudinit only supports raw and qcow2 atm:
6621             if ($dst_format eq 'qcow2') {
6622                 $name .= '.qcow2';
6623             } elsif ($dst_format ne 'raw') {
6624                 die "clone: unhandled format for cloudinit image\n";
6625             }
6626         }
6627         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6628         push @$newvollist, $newvolid;
6629
6630         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6631
6632         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6633         if (!$running || $snapname) {
6634             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6635         } else {
6636
6637             my $kvmver = get_running_qemu_version ($vmid);
6638             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6639                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6640                     if $drive->{iothread};
6641             }
6642
6643             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6644         }
6645     }
6646
6647     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6648
6649     my $disk = $drive;
6650     $disk->{format} = undef;
6651     $disk->{file} = $newvolid;
6652     $disk->{size} = $size;
6653
6654     return $disk;
6655 }
6656
6657 # this only works if VM is running
6658 sub get_current_qemu_machine {
6659     my ($vmid) = @_;
6660
6661     my $cmd = { execute => 'query-machines', arguments => {} };
6662     my $res = vm_qmp_command($vmid, $cmd);
6663
6664     my ($current, $default);
6665     foreach my $e (@$res) {
6666         $default = $e->{name} if $e->{'is-default'};
6667         $current = $e->{name} if $e->{'is-current'};
6668     }
6669
6670     # fallback to the default machine if current is not supported by qemu
6671     return $current || $default || 'pc';
6672 }
6673
6674 sub get_running_qemu_version {
6675     my ($vmid) = @_;
6676     my $cmd = { execute => 'query-version', arguments => {} };
6677     my $res = vm_qmp_command($vmid, $cmd);
6678     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6679 }
6680
6681 sub qemu_machine_feature_enabled {
6682     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6683
6684     my $current_major;
6685     my $current_minor;
6686
6687     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6688
6689         $current_major = $3;
6690         $current_minor = $4;
6691
6692     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6693
6694         $current_major = $1;
6695         $current_minor = $2;
6696     }
6697
6698     return 1 if $current_major > $version_major ||
6699                 ($current_major == $version_major &&
6700                  $current_minor >= $version_minor);
6701 }
6702
6703 sub qemu_machine_pxe {
6704     my ($vmid, $conf, $machine) = @_;
6705
6706     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6707
6708     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6709         $machine .= '.pxe';
6710     }
6711
6712     return $machine;
6713 }
6714
6715 sub qemu_use_old_bios_files {
6716     my ($machine_type) = @_;
6717
6718     return if !$machine_type;
6719
6720     my $use_old_bios_files = undef;
6721
6722     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6723         $machine_type = $1;
6724         $use_old_bios_files = 1;
6725     } else {
6726         my $kvmver = kvm_user_version();
6727         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6728         # load new efi bios files on migration. So this hack is required to allow
6729         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6730         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6731         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6732     }
6733
6734     return ($use_old_bios_files, $machine_type);
6735 }
6736
6737 sub create_efidisk($$$$$) {
6738     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6739
6740     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6741     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6742
6743     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6744     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6745     PVE::Storage::activate_volumes($storecfg, [$volid]);
6746
6747     my $path = PVE::Storage::path($storecfg, $volid);
6748     eval {
6749         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6750     };
6751     die "Copying EFI vars image failed: $@" if $@;
6752
6753     return ($volid, $vars_size);
6754 }
6755
6756 sub vm_iothreads_list {
6757     my ($vmid) = @_;
6758
6759     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6760
6761     my $iothreads = {};
6762     foreach my $iothread (@$res) {
6763         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6764     }
6765
6766     return $iothreads;
6767 }
6768
6769 sub scsihw_infos {
6770     my ($conf, $drive) = @_;
6771
6772     my $maxdev = 0;
6773
6774     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6775         $maxdev = 7;
6776     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6777         $maxdev = 1;
6778     } else {
6779         $maxdev = 256;
6780     }
6781
6782     my $controller = int($drive->{index} / $maxdev);
6783     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6784
6785     return ($maxdev, $controller, $controller_prefix);
6786 }
6787
6788 sub add_hyperv_enlightenments {
6789     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6790
6791     return if $winversion < 6;
6792     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6793
6794     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6795
6796     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6797         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6798         push @$cpuFlags , 'hv_vapic';
6799         push @$cpuFlags , 'hv_time';
6800     } else {
6801         push @$cpuFlags , 'hv_spinlocks=0xffff';
6802     }
6803
6804     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6805         push @$cpuFlags , 'hv_reset';
6806         push @$cpuFlags , 'hv_vpindex';
6807         push @$cpuFlags , 'hv_runtime';
6808     }
6809
6810     if ($winversion >= 7) {
6811         push @$cpuFlags , 'hv_relaxed';
6812
6813         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6814             push @$cpuFlags , 'hv_synic';
6815             push @$cpuFlags , 'hv_stimer';
6816         }
6817     }
6818 }
6819
6820 sub windows_version {
6821     my ($ostype) = @_;
6822
6823     return 0 if !$ostype;
6824
6825     my $winversion = 0;
6826
6827     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6828         $winversion = 5;
6829     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6830         $winversion = 6;
6831     } elsif ($ostype =~ m/^win(\d+)$/) {
6832         $winversion = $1;
6833     }
6834
6835     return $winversion;
6836 }
6837
6838 sub resolve_dst_disk_format {
6839         my ($storecfg, $storeid, $src_volname, $format) = @_;
6840         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6841
6842         if (!$format) {
6843             # if no target format is specified, use the source disk format as hint
6844             if ($src_volname) {
6845                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6846                 $format = qemu_img_format($scfg, $src_volname);
6847             } else {
6848                 return $defFormat;
6849             }
6850         }
6851
6852         # test if requested format is supported - else use default
6853         my $supported = grep { $_ eq $format } @$validFormats;
6854         $format = $defFormat if !$supported;
6855         return $format;
6856 }
6857
6858 sub resolve_first_disk {
6859     my $conf = shift;
6860     my @disks = PVE::QemuServer::valid_drive_names();
6861     my $firstdisk;
6862     foreach my $ds (reverse @disks) {
6863         next if !$conf->{$ds};
6864         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6865         next if PVE::QemuServer::drive_is_cdrom($disk);
6866         $firstdisk = $ds;
6867     }
6868     return $firstdisk;
6869 }
6870
6871 sub generate_uuid {
6872     my ($uuid, $uuid_str);
6873     UUID::generate($uuid);
6874     UUID::unparse($uuid, $uuid_str);
6875     return $uuid_str;
6876 }
6877
6878 sub generate_smbios1_uuid {
6879     return "uuid=".generate_uuid();
6880 }
6881
6882 sub nbd_stop {
6883     my ($vmid) = @_;
6884
6885     vm_mon_cmd($vmid, 'nbd-server-stop');
6886 }
6887
6888 # bash completion helper
6889
6890 sub complete_backup_archives {
6891     my ($cmdname, $pname, $cvalue) = @_;
6892
6893     my $cfg = PVE::Storage::config();
6894
6895     my $storeid;
6896
6897     if ($cvalue =~ m/^([^:]+):/) {
6898         $storeid = $1;
6899     }
6900
6901     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6902
6903     my $res = [];
6904     foreach my $id (keys %$data) {
6905         foreach my $item (@{$data->{$id}}) {
6906             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6907             push @$res, $item->{volid} if defined($item->{volid});
6908         }
6909     }
6910
6911     return $res;
6912 }
6913
6914 my $complete_vmid_full = sub {
6915     my ($running) = @_;
6916
6917     my $idlist = vmstatus();
6918
6919     my $res = [];
6920
6921     foreach my $id (keys %$idlist) {
6922         my $d = $idlist->{$id};
6923         if (defined($running)) {
6924             next if $d->{template};
6925             next if $running && $d->{status} ne 'running';
6926             next if !$running && $d->{status} eq 'running';
6927         }
6928         push @$res, $id;
6929
6930     }
6931     return $res;
6932 };
6933
6934 sub complete_vmid {
6935     return &$complete_vmid_full();
6936 }
6937
6938 sub complete_vmid_stopped {
6939     return &$complete_vmid_full(0);
6940 }
6941
6942 sub complete_vmid_running {
6943     return &$complete_vmid_full(1);
6944 }
6945
6946 sub complete_storage {
6947
6948     my $cfg = PVE::Storage::config();
6949     my $ids = $cfg->{ids};
6950
6951     my $res = [];
6952     foreach my $sid (keys %$ids) {
6953         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6954         next if !$ids->{$sid}->{content}->{images};
6955         push @$res, $sid;
6956     }
6957
6958     return $res;
6959 }
6960
6961 1;