PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $vga_fmt = {
 227     type => {
 228         description => "Select the VGA type.",
 229         type => 'string',
 230         default => 'std',
 231         optional => 1,
 232         default_key => 1,
 233         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 234     },
 235     memory => {
 236         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 237         type => 'integer',
 238         optional => 1,
 239         minimum => 4,
 240         maximum => 512,
 241     },
 242 };
 243
 244 my $confdesc = {
 245     onboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Specifies whether a VM will be started during system bootup.",
 249         default => 0,
 250     },
 251     autostart => {
 252         optional => 1,
 253         type => 'boolean',
 254         description => "Automatic restart after crash (currently ignored).",
 255         default => 0,
 256     },
 257     hotplug => {
 258         optional => 1,
 259         type => 'string', format => 'pve-hotplug-features',
 260         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 261         default => 'network,disk,usb',
 262     },
 263     reboot => {
 264         optional => 1,
 265         type => 'boolean',
 266         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 267         default => 1,
 268     },
 269     lock => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Lock/unlock the VM.",
 273         enum => [qw(migrate backup snapshot rollback)],
 274     },
 275     cpulimit => {
 276         optional => 1,
 277         type => 'number',
 278         description => "Limit of CPU usage.",
 279         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 280         minimum => 0,
 281         maximum => 128,
 282         default => 0,
 283     },
 284     cpuunits => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "CPU weight for a VM.",
 288         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 289         minimum => 2,
 290         maximum => 262144,
 291         default => 1024,
 292     },
 293     memory => {
 294         optional => 1,
 295         type => 'integer',
 296         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 297         minimum => 16,
 298         default => 512,
 299     },
 300     balloon => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 304         minimum => 0,
 305     },
 306     shares => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 310         minimum => 0,
 311         maximum => 50000,
 312         default => 1000,
 313     },
 314     keyboard => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 318                        "It should not be necessary to set it.",
 319         enum => PVE::Tools::kvmkeymaplist(),
 320         default => undef,
 321     },
 322     name => {
 323         optional => 1,
 324         type => 'string', format => 'dns-name',
 325         description => "Set a name for the VM. Only used on the configuration web interface.",
 326     },
 327     scsihw => {
 328         optional => 1,
 329         type => 'string',
 330         description => "SCSI controller model",
 331         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 332         default => 'lsi',
 333     },
 334     description => {
 335         optional => 1,
 336         type => 'string',
 337         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 338     },
 339     ostype => {
 340         optional => 1,
 341         type => 'string',
 342         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 343         description => "Specify guest operating system.",
 344         verbose_description => <<EODESC,
 345 Specify guest operating system. This is used to enable special
 346 optimization/features for specific operating systems:
 347
 348 [horizontal]
 349 other;; unspecified OS
 350 wxp;; Microsoft Windows XP
 351 w2k;; Microsoft Windows 2000
 352 w2k3;; Microsoft Windows 2003
 353 w2k8;; Microsoft Windows 2008
 354 wvista;; Microsoft Windows Vista
 355 win7;; Microsoft Windows 7
 356 win8;; Microsoft Windows 8/2012/2012r2
 357 win10;; Microsoft Windows 10/2016
 358 l24;; Linux 2.4 Kernel
 359 l26;; Linux 2.6/3.X Kernel
 360 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 361 EODESC
 362     },
 363     boot => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 367         pattern => '[acdn]{1,4}',
 368         default => 'cdn',
 369     },
 370     bootdisk => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-bootdisk',
 373         description => "Enable booting from specified disk.",
 374         pattern => '(ide|sata|scsi|virtio)\d+',
 375     },
 376     smp => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "The number of CPUs. Please use option -sockets instead.",
 380         minimum => 1,
 381         default => 1,
 382     },
 383     sockets => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPU sockets.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     cores => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of cores per socket.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     numa => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable NUMA.",
 401         default => 0,
 402     },
 403     hugepages => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Enable/disable hugepages memory.",
 407         enum => [qw(any 2 1024)],
 408     },
 409     vcpus => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "Number of hotplugged vcpus.",
 413         minimum => 1,
 414         default => 0,
 415     },
 416     acpi => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable ACPI.",
 420         default => 1,
 421     },
 422     agent => {
 423         optional => 1,
 424         description => "Enable/disable Qemu GuestAgent and its properties.",
 425         type => 'string',
 426         format => $agent_fmt,
 427     },
 428     kvm => {
 429         optional => 1,
 430         type => 'boolean',
 431         description => "Enable/disable KVM hardware virtualization.",
 432         default => 1,
 433     },
 434     tdf => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable time drift fix.",
 438         default => 0,
 439     },
 440     localtime => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 444     },
 445     freeze => {
 446         optional => 1,
 447         type => 'boolean',
 448         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 449     },
 450     vga => {
 451         optional => 1,
 452         type => 'string', format => $vga_fmt,
 453         description => "Configure the VGA hardware.",
 454         verbose_description => "Configure the VGA Hardware. If you want to use ".
 455             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 456             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 457             "is 'std' for all OS types besides some Windows versions (XP and " .
 458             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 459             "display server. For win* OS you can select how many independent " .
 460             "displays you want, Linux guests can add displays them self.\n".
 461             "You can also run without any graphic card, using a serial device as terminal.",
 462     },
 463     watchdog => {
 464         optional => 1,
 465         type => 'string', format => 'pve-qm-watchdog',
 466         description => "Create a virtual hardware watchdog device.",
 467         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 468             " (by a guest action), the watchdog must be periodically polled " .
 469             "by an agent inside the guest or else the watchdog will reset " .
 470             "the guest (or execute the respective action specified)",
 471     },
 472     startdate => {
 473         optional => 1,
 474         type => 'string',
 475         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 476         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 477         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 478         default => 'now',
 479     },
 480     startup =>  get_standard_option('pve-startup-order'),
 481     template => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable Template.",
 485         default => 0,
 486     },
 487     args => {
 488         optional => 1,
 489         type => 'string',
 490         description => "Arbitrary arguments passed to kvm.",
 491         verbose_description => <<EODESCR,
 492 Arbitrary arguments passed to kvm, for example:
 493
 494 args: -no-reboot -no-hpet
 495
 496 NOTE: this option is for experts only.
 497 EODESCR
 498     },
 499     tablet => {
 500         optional => 1,
 501         type => 'boolean',
 502         default => 1,
 503         description => "Enable/disable the USB tablet device.",
 504         verbose_description => "Enable/disable the USB tablet device. This device is " .
 505             "usually needed to allow absolute mouse positioning with VNC. " .
 506             "Else the mouse runs out of sync with normal VNC clients. " .
 507             "If you're running lots of console-only guests on one host, " .
 508             "you may consider disabling this to save some context switches. " .
 509             "This is turned off by default if you use spice (-vga=qxl).",
 510     },
 511     migrate_speed => {
 512         optional => 1,
 513         type => 'integer',
 514         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 515         minimum => 0,
 516         default => 0,
 517     },
 518     migrate_downtime => {
 519         optional => 1,
 520         type => 'number',
 521         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 522         minimum => 0,
 523         default => 0.1,
 524     },
 525     cdrom => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-ide',
 528         typetext => '<volume>',
 529         description => "This is an alias for option -ide2",
 530     },
 531     cpu => {
 532         optional => 1,
 533         description => "Emulated CPU type.",
 534         type => 'string',
 535         format => $cpu_fmt,
 536     },
 537     parent => get_standard_option('pve-snapshot-name', {
 538         optional => 1,
 539         description => "Parent snapshot name. This is used internally, and should not be modified.",
 540     }),
 541     snaptime => {
 542         optional => 1,
 543         description => "Timestamp for snapshots.",
 544         type => 'integer',
 545         minimum => 0,
 546     },
 547     vmstate => {
 548         optional => 1,
 549         type => 'string', format => 'pve-volume-id',
 550         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 551     },
 552     vmstatestorage => get_standard_option('pve-storage-id', {
 553         description => "Default storage for VM state volumes/files.",
 554         optional => 1,
 555     }),
 556     runningmachine => get_standard_option('pve-qemu-machine', {
 557         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 558     }),
 559     machine => get_standard_option('pve-qemu-machine'),
 560     smbios1 => {
 561         description => "Specify SMBIOS type 1 fields.",
 562         type => 'string', format => 'pve-qm-smbios1',
 563         maxLength => 256,
 564         optional => 1,
 565     },
 566     protection => {
 567         optional => 1,
 568         type => 'boolean',
 569         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 570         default => 0,
 571     },
 572     bios => {
 573         optional => 1,
 574         type => 'string',
 575         enum => [ qw(seabios ovmf) ],
 576         description => "Select BIOS implementation.",
 577         default => 'seabios',
 578     },
 579     vmgenid => {
 580         type => 'string',
 581         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 582         format_description => 'UUID',
 583         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 584         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 585             " 128-bit integer value identifier to the guest OS. This allows to".
 586             " notify the guest operating system when the virtual machine is".
 587             " executed with a different configuration (e.g. snapshot execution".
 588             " or creation from a template). The guest operating system notices".
 589             " the change, and is then able to react as appropriate by marking".
 590             " its copies of distributed databases as dirty, re-initializing its".
 591             " random number generator, etc.\n".
 592             "Note that auto-creation only works when done throug API/CLI create".
 593             " or update methods, but not when manually editing the config file.",
 594         default => "1 (autogenerated)",
 595         optional => 1,
 596     },
 597 };
 598
 599 my $confdesc_cloudinit = {
 600     citype => {
 601         optional => 1,
 602         type => 'string',
 603         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 604         enum => ['configdrive2', 'nocloud'],
 605     },
 606     ciuser => {
 607         optional => 1,
 608         type => 'string',
 609         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 610     },
 611     cipassword => {
 612         optional => 1,
 613         type => 'string',
 614         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 615     },
 616     searchdomain => {
 617         optional => 1,
 618         type => 'string',
 619         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 620     },
 621     nameserver => {
 622         optional => 1,
 623         type => 'string', format => 'address-list',
 624         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 625     },
 626     sshkeys => {
 627         optional => 1,
 628         type => 'string',
 629         format => 'urlencoded',
 630         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 631     },
 632 };
 633
 634 # what about other qemu settings ?
 635 #cpu => 'string',
 636 #machine => 'string',
 637 #fda => 'file',
 638 #fdb => 'file',
 639 #mtdblock => 'file',
 640 #sd => 'file',
 641 #pflash => 'file',
 642 #snapshot => 'bool',
 643 #bootp => 'file',
 644 ##tftp => 'dir',
 645 ##smb => 'dir',
 646 #kernel => 'file',
 647 #append => 'string',
 648 #initrd => 'file',
 649 ##soundhw => 'string',
 650
 651 while (my ($k, $v) = each %$confdesc) {
 652     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 653 }
 654
 655 my $MAX_IDE_DISKS = 4;
 656 my $MAX_SCSI_DISKS = 14;
 657 my $MAX_VIRTIO_DISKS = 16;
 658 my $MAX_SATA_DISKS = 6;
 659 my $MAX_USB_DEVICES = 5;
 660 my $MAX_NETS = 32;
 661 my $MAX_UNUSED_DISKS = 8;
 662 my $MAX_HOSTPCI_DEVICES = 4;
 663 my $MAX_SERIAL_PORTS = 4;
 664 my $MAX_PARALLEL_PORTS = 3;
 665 my $MAX_NUMA = 8;
 666
 667 my $numa_fmt = {
 668     cpus => {
 669         type => "string",
 670         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 671         description => "CPUs accessing this NUMA node.",
 672         format_description => "id[-id];...",
 673     },
 674     memory => {
 675         type => "number",
 676         description => "Amount of memory this NUMA node provides.",
 677         optional => 1,
 678     },
 679     hostnodes => {
 680         type => "string",
 681         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 682         description => "Host NUMA nodes to use.",
 683         format_description => "id[-id];...",
 684         optional => 1,
 685     },
 686     policy => {
 687         type => 'string',
 688         enum => [qw(preferred bind interleave)],
 689         description => "NUMA allocation policy.",
 690         optional => 1,
 691     },
 692 };
 693 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 694 my $numadesc = {
 695     optional => 1,
 696     type => 'string', format => $numa_fmt,
 697     description => "NUMA topology.",
 698 };
 699 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 700
 701 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 702     $confdesc->{"numa$i"} = $numadesc;
 703 }
 704
 705 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 706                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 707                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 708 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 709
 710 my $net_fmt_bridge_descr = <<__EOD__;
 711 Bridge to attach the network device to. The Proxmox VE standard bridge
 712 is called 'vmbr0'.
 713
 714 If you do not specify a bridge, we create a kvm user (NATed) network
 715 device, which provides DHCP and DNS services. The following addresses
 716 are used:
 717
 718  10.0.2.2   Gateway
 719  10.0.2.3   DNS Server
 720  10.0.2.4   SMB Server
 721
 722 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 723 __EOD__
 724
 725 my $net_fmt = {
 726     macaddr => {
 727         type => 'string',
 728         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 729         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 730         format_description => "XX:XX:XX:XX:XX:XX",
 731         optional => 1,
 732     },
 733     model => {
 734         type => 'string',
 735         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 736         enum => $nic_model_list,
 737         default_key => 1,
 738     },
 739     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 740     bridge => {
 741         type => 'string',
 742         description => $net_fmt_bridge_descr,
 743         format_description => 'bridge',
 744         optional => 1,
 745     },
 746     queues => {
 747         type => 'integer',
 748         minimum => 0, maximum => 16,
 749         description => 'Number of packet queues to be used on the device.',
 750         optional => 1,
 751     },
 752     rate => {
 753         type => 'number',
 754         minimum => 0,
 755         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 756         optional => 1,
 757     },
 758     tag => {
 759         type => 'integer',
 760         minimum => 1, maximum => 4094,
 761         description => 'VLAN tag to apply to packets on this interface.',
 762         optional => 1,
 763     },
 764     trunks => {
 765         type => 'string',
 766         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 767         description => 'VLAN trunks to pass through this interface.',
 768         format_description => 'vlanid[;vlanid...]',
 769         optional => 1,
 770     },
 771     firewall => {
 772         type => 'boolean',
 773         description => 'Whether this interface should be protected by the firewall.',
 774         optional => 1,
 775     },
 776     link_down => {
 777         type => 'boolean',
 778         description => 'Whether this interface should be disconnected (like pulling the plug).',
 779         optional => 1,
 780     },
 781 };
 782
 783 my $netdesc = {
 784     optional => 1,
 785     type => 'string', format => $net_fmt,
 786     description => "Specify network devices.",
 787 };
 788
 789 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 790
 791 my $ipconfig_fmt = {
 792     ip => {
 793         type => 'string',
 794         format => 'pve-ipv4-config',
 795         format_description => 'IPv4Format/CIDR',
 796         description => 'IPv4 address in CIDR format.',
 797         optional => 1,
 798         default => 'dhcp',
 799     },
 800     gw => {
 801         type => 'string',
 802         format => 'ipv4',
 803         format_description => 'GatewayIPv4',
 804         description => 'Default gateway for IPv4 traffic.',
 805         optional => 1,
 806         requires => 'ip',
 807     },
 808     ip6 => {
 809         type => 'string',
 810         format => 'pve-ipv6-config',
 811         format_description => 'IPv6Format/CIDR',
 812         description => 'IPv6 address in CIDR format.',
 813         optional => 1,
 814         default => 'dhcp',
 815     },
 816     gw6 => {
 817         type => 'string',
 818         format => 'ipv6',
 819         format_description => 'GatewayIPv6',
 820         description => 'Default gateway for IPv6 traffic.',
 821         optional => 1,
 822         requires => 'ip6',
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 826 my $ipconfigdesc = {
 827     optional => 1,
 828     type => 'string', format => 'pve-qm-ipconfig',
 829     description => <<'EODESCR',
 830 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 831
 832 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 833
 834 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 835 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 836
 837 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 838 EODESCR
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 841
 842 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 843     $confdesc->{"net$i"} = $netdesc;
 844     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 845 }
 846
 847 foreach my $key (keys %$confdesc_cloudinit) {
 848     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 849 }
 850
 851 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 852 sub verify_volume_id_or_qm_path {
 853     my ($volid, $noerr) = @_;
 854
 855     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 856         return $volid;
 857     }
 858
 859     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 860     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 861     if ($@) {
 862         return undef if $noerr;
 863         die $@;
 864     }
 865     return $volid;
 866 }
 867
 868 my $drivename_hash;
 869
 870 my %drivedesc_base = (
 871     volume => { alias => 'file' },
 872     file => {
 873         type => 'string',
 874         format => 'pve-volume-id-or-qm-path',
 875         default_key => 1,
 876         format_description => 'volume',
 877         description => "The drive's backing volume.",
 878     },
 879     media => {
 880         type => 'string',
 881         enum => [qw(cdrom disk)],
 882         description => "The drive's media type.",
 883         default => 'disk',
 884         optional => 1
 885     },
 886     cyls => {
 887         type => 'integer',
 888         description => "Force the drive's physical geometry to have a specific cylinder count.",
 889         optional => 1
 890     },
 891     heads => {
 892         type => 'integer',
 893         description => "Force the drive's physical geometry to have a specific head count.",
 894         optional => 1
 895     },
 896     secs => {
 897         type => 'integer',
 898         description => "Force the drive's physical geometry to have a specific sector count.",
 899         optional => 1
 900     },
 901     trans => {
 902         type => 'string',
 903         enum => [qw(none lba auto)],
 904         description => "Force disk geometry bios translation mode.",
 905         optional => 1,
 906     },
 907     snapshot => {
 908         type => 'boolean',
 909         description => "Controls qemu's snapshot mode feature."
 910             . " If activated, changes made to the disk are temporary and will"
 911             . " be discarded when the VM is shutdown.",
 912         optional => 1,
 913     },
 914     cache => {
 915         type => 'string',
 916         enum => [qw(none writethrough writeback unsafe directsync)],
 917         description => "The drive's cache mode",
 918         optional => 1,
 919     },
 920     format => get_standard_option('pve-qm-image-format'),
 921     size => {
 922         type => 'string',
 923         format => 'disk-size',
 924         format_description => 'DiskSize',
 925         description => "Disk size. This is purely informational and has no effect.",
 926         optional => 1,
 927     },
 928     backup => {
 929         type => 'boolean',
 930         description => "Whether the drive should be included when making backups.",
 931         optional => 1,
 932     },
 933     replicate => {
 934         type => 'boolean',
 935         description => 'Whether the drive should considered for replication jobs.',
 936         optional => 1,
 937         default => 1,
 938     },
 939     rerror => {
 940         type => 'string',
 941         enum => [qw(ignore report stop)],
 942         description => 'Read error action.',
 943         optional => 1,
 944     },
 945     werror => {
 946         type => 'string',
 947         enum => [qw(enospc ignore report stop)],
 948         description => 'Write error action.',
 949         optional => 1,
 950     },
 951     aio => {
 952         type => 'string',
 953         enum => [qw(native threads)],
 954         description => 'AIO type to use.',
 955         optional => 1,
 956     },
 957     discard => {
 958         type => 'string',
 959         enum => [qw(ignore on)],
 960         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 961         optional => 1,
 962     },
 963     detect_zeroes => {
 964         type => 'boolean',
 965         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 966         optional => 1,
 967     },
 968     serial => {
 969         type => 'string',
 970         format => 'urlencoded',
 971         format_description => 'serial',
 972         maxLength => 20*3, # *3 since it's %xx url enoded
 973         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 974         optional => 1,
 975     },
 976     shared => {
 977         type => 'boolean',
 978         description => 'Mark this locally-managed volume as available on all nodes',
 979         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 980         optional => 1,
 981         default => 0,
 982     }
 983 );
 984
 985 my %iothread_fmt = ( iothread => {
 986         type => 'boolean',
 987         description => "Whether to use iothreads for this drive",
 988         optional => 1,
 989 });
 990
 991 my %model_fmt = (
 992     model => {
 993         type => 'string',
 994         format => 'urlencoded',
 995         format_description => 'model',
 996         maxLength => 40*3, # *3 since it's %xx url enoded
 997         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 998         optional => 1,
 999     },
1000 );
1001
1002 my %queues_fmt = (
1003     queues => {
1004         type => 'integer',
1005         description => "Number of queues.",
1006         minimum => 2,
1007         optional => 1
1008     }
1009 );
1010
1011 my %scsiblock_fmt = (
1012     scsiblock => {
1013         type => 'boolean',
1014         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1015         optional => 1,
1016         default => 0,
1017     },
1018 );
1019
1020 my %ssd_fmt = (
1021     ssd => {
1022         type => 'boolean',
1023         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1024         optional => 1,
1025     },
1026 );
1027
1028 my $add_throttle_desc = sub {
1029     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1030     my $d = {
1031         type => $type,
1032         format_description => $unit,
1033         description => "Maximum $what in $longunit.",
1034         optional => 1,
1035     };
1036     $d->{minimum} = $minimum if defined($minimum);
1037     $drivedesc_base{$key} = $d;
1038 };
1039 # throughput: (leaky bucket)
1040 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1041 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1042 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1043 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1044 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1045 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1046 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1047 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1048 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1049
1050 # pools: (pool of IO before throttling starts taking effect)
1051 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1052 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1053 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1054 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1055 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1056 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1057
1058 # burst lengths
1059 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1060 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1061 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1062 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1063 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1064 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1065
1066 # legacy support
1067 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1068 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1069 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1070 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1071
1072 my $ide_fmt = {
1073     %drivedesc_base,
1074     %model_fmt,
1075     %ssd_fmt,
1076 };
1077 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1078
1079 my $idedesc = {
1080     optional => 1,
1081     type => 'string', format => $ide_fmt,
1082     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1083 };
1084 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1085
1086 my $scsi_fmt = {
1087     %drivedesc_base,
1088     %iothread_fmt,
1089     %queues_fmt,
1090     %scsiblock_fmt,
1091     %ssd_fmt,
1092 };
1093 my $scsidesc = {
1094     optional => 1,
1095     type => 'string', format => $scsi_fmt,
1096     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1099
1100 my $sata_fmt = {
1101     %drivedesc_base,
1102     %ssd_fmt,
1103 };
1104 my $satadesc = {
1105     optional => 1,
1106     type => 'string', format => $sata_fmt,
1107     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1110
1111 my $virtio_fmt = {
1112     %drivedesc_base,
1113     %iothread_fmt,
1114 };
1115 my $virtiodesc = {
1116     optional => 1,
1117     type => 'string', format => $virtio_fmt,
1118     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1119 };
1120 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1121
1122 my $alldrive_fmt = {
1123     %drivedesc_base,
1124     %iothread_fmt,
1125     %model_fmt,
1126     %queues_fmt,
1127     %scsiblock_fmt,
1128     %ssd_fmt,
1129 };
1130
1131 my $efidisk_fmt = {
1132     volume => { alias => 'file' },
1133     file => {
1134         type => 'string',
1135         format => 'pve-volume-id-or-qm-path',
1136         default_key => 1,
1137         format_description => 'volume',
1138         description => "The drive's backing volume.",
1139     },
1140     format => get_standard_option('pve-qm-image-format'),
1141     size => {
1142         type => 'string',
1143         format => 'disk-size',
1144         format_description => 'DiskSize',
1145         description => "Disk size. This is purely informational and has no effect.",
1146         optional => 1,
1147     },
1148 };
1149
1150 my $efidisk_desc = {
1151     optional => 1,
1152     type => 'string', format => $efidisk_fmt,
1153     description => "Configure a Disk for storing EFI vars",
1154 };
1155
1156 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1157
1158 my $usb_fmt = {
1159     host => {
1160         default_key => 1,
1161         type => 'string', format => 'pve-qm-usb-device',
1162         format_description => 'HOSTUSBDEVICE|spice',
1163         description => <<EODESCR,
1164 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1165
1166  'bus-port(.port)*' (decimal numbers) or
1167  'vendor_id:product_id' (hexadeciaml numbers) or
1168  'spice'
1169
1170 You can use the 'lsusb -t' command to list existing usb devices.
1171
1172 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1173
1174 The value 'spice' can be used to add a usb redirection devices for spice.
1175 EODESCR
1176     },
1177     usb3 => {
1178         optional => 1,
1179         type => 'boolean',
1180         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1181         default => 0,
1182     },
1183 };
1184
1185 my $usbdesc = {
1186     optional => 1,
1187     type => 'string', format => $usb_fmt,
1188     description => "Configure an USB device (n is 0 to 4).",
1189 };
1190 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1191
1192 # NOTE: the match-groups of this regex are used in parse_hostpci
1193 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1194 my $hostpci_fmt = {
1195     host => {
1196         default_key => 1,
1197         type => 'string',
1198         pattern => qr/$PCIRE(;$PCIRE)*/,
1199         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1200         description => <<EODESCR,
1201 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1202 of PCI virtual functions of the host. HOSTPCIID syntax is:
1203
1204 'bus:dev.func' (hexadecimal numbers)
1205
1206 You can us the 'lspci' command to list existing PCI devices.
1207 EODESCR
1208     },
1209     rombar => {
1210         type => 'boolean',
1211         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1212         optional => 1,
1213         default => 1,
1214     },
1215     romfile => {
1216         type => 'string',
1217         pattern => '[^,;]+',
1218         format_description => 'string',
1219         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1220         optional => 1,
1221     },
1222     pcie => {
1223         type => 'boolean',
1224         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1225         optional => 1,
1226         default => 0,
1227     },
1228     'x-vga' => {
1229         type => 'boolean',
1230         description =>  "Enable vfio-vga device support.",
1231         optional => 1,
1232         default => 0,
1233     },
1234 };
1235 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1236
1237 my $hostpcidesc = {
1238         optional => 1,
1239         type => 'string', format => 'pve-qm-hostpci',
1240         description => "Map host PCI devices into guest.",
1241         verbose_description =>  <<EODESCR,
1242 Map host PCI devices into guest.
1243
1244 NOTE: This option allows direct access to host hardware. So it is no longer
1245 possible to migrate such machines - use with special care.
1246
1247 CAUTION: Experimental! User reported problems with this option.
1248 EODESCR
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1251
1252 my $serialdesc = {
1253         optional => 1,
1254         type => 'string',
1255         pattern => '(/dev/.+|socket)',
1256         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1257         verbose_description =>  <<EODESCR,
1258 Create a serial device inside the VM (n is 0 to 3), and pass through a
1259 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1260 host side (use 'qm terminal' to open a terminal connection).
1261
1262 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1263
1264 CAUTION: Experimental! User reported problems with this option.
1265 EODESCR
1266 };
1267
1268 my $paralleldesc= {
1269         optional => 1,
1270         type => 'string',
1271         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1272         description =>  "Map host parallel devices (n is 0 to 2).",
1273         verbose_description =>  <<EODESCR,
1274 Map host parallel devices (n is 0 to 2).
1275
1276 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1283     $confdesc->{"parallel$i"} = $paralleldesc;
1284 }
1285
1286 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1287     $confdesc->{"serial$i"} = $serialdesc;
1288 }
1289
1290 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1291     $confdesc->{"hostpci$i"} = $hostpcidesc;
1292 }
1293
1294 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1295     $drivename_hash->{"ide$i"} = 1;
1296     $confdesc->{"ide$i"} = $idedesc;
1297 }
1298
1299 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1300     $drivename_hash->{"sata$i"} = 1;
1301     $confdesc->{"sata$i"} = $satadesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1305     $drivename_hash->{"scsi$i"} = 1;
1306     $confdesc->{"scsi$i"} = $scsidesc ;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1310     $drivename_hash->{"virtio$i"} = 1;
1311     $confdesc->{"virtio$i"} = $virtiodesc;
1312 }
1313
1314 $drivename_hash->{efidisk0} = 1;
1315 $confdesc->{efidisk0} = $efidisk_desc;
1316
1317 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1318     $confdesc->{"usb$i"} = $usbdesc;
1319 }
1320
1321 my $unuseddesc = {
1322     optional => 1,
1323     type => 'string', format => 'pve-volume-id',
1324     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1325 };
1326
1327 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1328     $confdesc->{"unused$i"} = $unuseddesc;
1329 }
1330
1331 my $kvm_api_version = 0;
1332
1333 sub kvm_version {
1334
1335     return $kvm_api_version if $kvm_api_version;
1336
1337     my $fh = IO::File->new("</dev/kvm") ||
1338         return 0;
1339
1340     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1341         $kvm_api_version = $v;
1342     }
1343
1344     $fh->close();
1345
1346     return  $kvm_api_version;
1347 }
1348
1349 my $kvm_user_version;
1350
1351 sub kvm_user_version {
1352
1353     return $kvm_user_version if $kvm_user_version;
1354
1355     $kvm_user_version = 'unknown';
1356
1357     my $code = sub {
1358         my $line = shift;
1359         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1360             $kvm_user_version = $2;
1361         }
1362     };
1363
1364     eval { run_command("kvm -version", outfunc => $code); };
1365     warn $@ if $@;
1366
1367     return $kvm_user_version;
1368
1369 }
1370
1371 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1372
1373 sub valid_drive_names {
1374     # order is important - used to autoselect boot disk
1375     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1376             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1377             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1378             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1379             'efidisk0');
1380 }
1381
1382 sub is_valid_drivename {
1383     my $dev = shift;
1384
1385     return defined($drivename_hash->{$dev});
1386 }
1387
1388 sub option_exists {
1389     my $key = shift;
1390     return defined($confdesc->{$key});
1391 }
1392
1393 sub nic_models {
1394     return $nic_model_list;
1395 }
1396
1397 sub os_list_description {
1398
1399     return {
1400         other => 'Other',
1401         wxp => 'Windows XP',
1402         w2k => 'Windows 2000',
1403         w2k3 =>, 'Windows 2003',
1404         w2k8 => 'Windows 2008',
1405         wvista => 'Windows Vista',
1406         win7 => 'Windows 7',
1407         win8 => 'Windows 8/2012',
1408         win10 => 'Windows 10/2016',
1409         l24 => 'Linux 2.4',
1410         l26 => 'Linux 2.6',
1411     };
1412 }
1413
1414 my $cdrom_path;
1415
1416 sub get_cdrom_path {
1417
1418     return  $cdrom_path if $cdrom_path;
1419
1420     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1421     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1422     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1423 }
1424
1425 sub get_iso_path {
1426     my ($storecfg, $vmid, $cdrom) = @_;
1427
1428     if ($cdrom eq 'cdrom') {
1429         return get_cdrom_path();
1430     } elsif ($cdrom eq 'none') {
1431         return '';
1432     } elsif ($cdrom =~ m|^/|) {
1433         return $cdrom;
1434     } else {
1435         return PVE::Storage::path($storecfg, $cdrom);
1436     }
1437 }
1438
1439 # try to convert old style file names to volume IDs
1440 sub filename_to_volume_id {
1441     my ($vmid, $file, $media) = @_;
1442
1443      if (!($file eq 'none' || $file eq 'cdrom' ||
1444           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1445
1446         return undef if $file =~ m|/|;
1447
1448         if ($media && $media eq 'cdrom') {
1449             $file = "local:iso/$file";
1450         } else {
1451             $file = "local:$vmid/$file";
1452         }
1453     }
1454
1455     return $file;
1456 }
1457
1458 sub verify_media_type {
1459     my ($opt, $vtype, $media) = @_;
1460
1461     return if !$media;
1462
1463     my $etype;
1464     if ($media eq 'disk') {
1465         $etype = 'images';
1466     } elsif ($media eq 'cdrom') {
1467         $etype = 'iso';
1468     } else {
1469         die "internal error";
1470     }
1471
1472     return if ($vtype eq $etype);
1473
1474     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1475 }
1476
1477 sub cleanup_drive_path {
1478     my ($opt, $storecfg, $drive) = @_;
1479
1480     # try to convert filesystem paths to volume IDs
1481
1482     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1483         ($drive->{file} !~ m|^/dev/.+|) &&
1484         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1485         ($drive->{file} !~ m/^\d+$/)) {
1486         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1487         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1488         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1489         verify_media_type($opt, $vtype, $drive->{media});
1490         $drive->{file} = $volid;
1491     }
1492
1493     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1494 }
1495
1496 sub parse_hotplug_features {
1497     my ($data) = @_;
1498
1499     my $res = {};
1500
1501     return $res if $data eq '0';
1502
1503     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1504
1505     foreach my $feature (PVE::Tools::split_list($data)) {
1506         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1507             $res->{$1} = 1;
1508         } else {
1509             die "invalid hotplug feature '$feature'\n";
1510         }
1511     }
1512     return $res;
1513 }
1514
1515 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1516 sub pve_verify_hotplug_features {
1517     my ($value, $noerr) = @_;
1518
1519     return $value if parse_hotplug_features($value);
1520
1521     return undef if $noerr;
1522
1523     die "unable to parse hotplug option\n";
1524 }
1525
1526 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1527 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1528 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1529 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1530 #        [,iothread=on][,serial=serial][,model=model]
1531
1532 sub parse_drive {
1533     my ($key, $data) = @_;
1534
1535     my ($interface, $index);
1536
1537     if ($key =~ m/^([^\d]+)(\d+)$/) {
1538         $interface = $1;
1539         $index = $2;
1540     } else {
1541         return undef;
1542     }
1543
1544     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1545                                      : $confdesc->{$key}->{format};
1546     if (!$desc) {
1547         warn "invalid drive key: $key\n";
1548         return undef;
1549     }
1550     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1551     return undef if !$res;
1552     $res->{interface} = $interface;
1553     $res->{index} = $index;
1554
1555     my $error = 0;
1556     foreach my $opt (qw(bps bps_rd bps_wr)) {
1557         if (my $bps = defined(delete $res->{$opt})) {
1558             if (defined($res->{"m$opt"})) {
1559                 warn "both $opt and m$opt specified\n";
1560                 ++$error;
1561                 next;
1562             }
1563             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1564         }
1565     }
1566
1567     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1568     for my $requirement (
1569         [mbps_max => 'mbps'],
1570         [mbps_rd_max => 'mbps_rd'],
1571         [mbps_wr_max => 'mbps_wr'],
1572         [miops_max => 'miops'],
1573         [miops_rd_max => 'miops_rd'],
1574         [miops_wr_max => 'miops_wr'],
1575         [bps_max_length => 'mbps_max'],
1576         [bps_rd_max_length => 'mbps_rd_max'],
1577         [bps_wr_max_length => 'mbps_wr_max'],
1578         [iops_max_length => 'iops_max'],
1579         [iops_rd_max_length => 'iops_rd_max'],
1580         [iops_wr_max_length => 'iops_wr_max']) {
1581         my ($option, $requires) = @$requirement;
1582         if ($res->{$option} && !$res->{$requires}) {
1583             warn "$option requires $requires\n";
1584             ++$error;
1585         }
1586     }
1587
1588     return undef if $error;
1589
1590     return undef if $res->{mbps_rd} && $res->{mbps};
1591     return undef if $res->{mbps_wr} && $res->{mbps};
1592     return undef if $res->{iops_rd} && $res->{iops};
1593     return undef if $res->{iops_wr} && $res->{iops};
1594
1595     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1596         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1597         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1598         return undef if $res->{interface} eq 'virtio';
1599     }
1600
1601     if (my $size = $res->{size}) {
1602         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1603     }
1604
1605     return $res;
1606 }
1607
1608 sub print_drive {
1609     my ($vmid, $drive) = @_;
1610     my $data = { %$drive };
1611     delete $data->{$_} for qw(index interface);
1612     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1613 }
1614
1615 sub scsi_inquiry {
1616     my($fh, $noerr) = @_;
1617
1618     my $SG_IO = 0x2285;
1619     my $SG_GET_VERSION_NUM = 0x2282;
1620
1621     my $versionbuf = "\x00" x 8;
1622     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1623     if (!$ret) {
1624         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1625         return undef;
1626     }
1627     my $version = unpack("I", $versionbuf);
1628     if ($version < 30000) {
1629         die "scsi generic interface too old\n"  if !$noerr;
1630         return undef;
1631     }
1632
1633     my $buf = "\x00" x 36;
1634     my $sensebuf = "\x00" x 8;
1635     my $cmd = pack("C x3 C x1", 0x12, 36);
1636
1637     # see /usr/include/scsi/sg.h
1638     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1639
1640     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1641                       length($sensebuf), 0, length($buf), $buf,
1642                       $cmd, $sensebuf, 6000);
1643
1644     $ret = ioctl($fh, $SG_IO, $packet);
1645     if (!$ret) {
1646         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1647         return undef;
1648     }
1649
1650     my @res = unpack($sg_io_hdr_t, $packet);
1651     if ($res[17] || $res[18]) {
1652         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1653         return undef;
1654     }
1655
1656     my $res = {};
1657     (my $byte0, my $byte1, $res->{vendor},
1658      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1659
1660     $res->{removable} = $byte1 & 128 ? 1 : 0;
1661     $res->{type} = $byte0 & 31;
1662
1663     return $res;
1664 }
1665
1666 sub path_is_scsi {
1667     my ($path) = @_;
1668
1669     my $fh = IO::File->new("+<$path") || return undef;
1670     my $res = scsi_inquiry($fh, 1);
1671     close($fh);
1672
1673     return $res;
1674 }
1675
1676 sub machine_type_is_q35 {
1677     my ($conf) = @_;
1678
1679     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1680 }
1681
1682 sub print_tabletdevice_full {
1683     my ($conf) = @_;
1684
1685     my $q35 = machine_type_is_q35($conf);
1686
1687     # we use uhci for old VMs because tablet driver was buggy in older qemu
1688     my $usbbus = $q35 ? "ehci" : "uhci";
1689
1690     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1691 }
1692
1693 sub print_drivedevice_full {
1694     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1695
1696     my $device = '';
1697     my $maxdev = 0;
1698
1699     if ($drive->{interface} eq 'virtio') {
1700         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1701         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1702         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1703     } elsif ($drive->{interface} eq 'scsi') {
1704
1705         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1706         my $unit = $drive->{index} % $maxdev;
1707         my $devicetype = 'hd';
1708         my $path = '';
1709         if (drive_is_cdrom($drive)) {
1710             $devicetype = 'cd';
1711         } else {
1712             if ($drive->{file} =~ m|^/|) {
1713                 $path = $drive->{file};
1714                 if (my $info = path_is_scsi($path)) {
1715                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1716                         $devicetype = 'block';
1717                     } elsif ($info->{type} == 1) { # tape
1718                         $devicetype = 'generic';
1719                     }
1720                 }
1721             } else {
1722                  $path = PVE::Storage::path($storecfg, $drive->{file});
1723             }
1724
1725             if($path =~ m/^iscsi\:\/\//){
1726                 $devicetype = 'generic';
1727             }
1728         }
1729
1730         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1731             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1732         } else {
1733             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1734         }
1735
1736         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1737             $device .= ",rotation_rate=1";
1738         }
1739
1740     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1741         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1742         my $controller = int($drive->{index} / $maxdev);
1743         my $unit = $drive->{index} % $maxdev;
1744         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1745
1746         $device = "ide-$devicetype";
1747         if ($drive->{interface} eq 'ide') {
1748             $device .= ",bus=ide.$controller,unit=$unit";
1749         } else {
1750             $device .= ",bus=ahci$controller.$unit";
1751         }
1752         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1753
1754         if ($devicetype eq 'hd') {
1755             if (my $model = $drive->{model}) {
1756                 $model = URI::Escape::uri_unescape($model);
1757                 $device .= ",model=$model";
1758             }
1759             if ($drive->{ssd}) {
1760                 $device .= ",rotation_rate=1";
1761             }
1762         }
1763     } elsif ($drive->{interface} eq 'usb') {
1764         die "implement me";
1765         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1766     } else {
1767         die "unsupported interface type";
1768     }
1769
1770     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1771
1772     if (my $serial = $drive->{serial}) {
1773         $serial = URI::Escape::uri_unescape($serial);
1774         $device .= ",serial=$serial";
1775     }
1776
1777
1778     return $device;
1779 }
1780
1781 sub get_initiator_name {
1782     my $initiator;
1783
1784     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1785     while (defined(my $line = <$fh>)) {
1786         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1787         $initiator = $1;
1788         last;
1789     }
1790     $fh->close();
1791
1792     return $initiator;
1793 }
1794
1795 sub print_drive_full {
1796     my ($storecfg, $vmid, $drive) = @_;
1797
1798     my $path;
1799     my $volid = $drive->{file};
1800     my $format;
1801
1802     if (drive_is_cdrom($drive)) {
1803         $path = get_iso_path($storecfg, $vmid, $volid);
1804     } else {
1805         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1806         if ($storeid) {
1807             $path = PVE::Storage::path($storecfg, $volid);
1808             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1809             $format = qemu_img_format($scfg, $volname);
1810         } else {
1811             $path = $volid;
1812             $format = "raw";
1813         }
1814    }
1815
1816     my $opts = '';
1817     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1818     foreach my $o (@qemu_drive_options) {
1819         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1820     }
1821
1822     # snapshot only accepts on|off
1823     if (defined($drive->{snapshot})) {
1824         my $v = $drive->{snapshot} ? 'on' : 'off';
1825         $opts .= ",snapshot=$v";
1826     }
1827
1828     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1829         my ($dir, $qmpname) = @$type;
1830         if (my $v = $drive->{"mbps$dir"}) {
1831             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1832         }
1833         if (my $v = $drive->{"mbps${dir}_max"}) {
1834             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1835         }
1836         if (my $v = $drive->{"bps${dir}_max_length"}) {
1837             $opts .= ",throttling.bps$qmpname-max-length=$v";
1838         }
1839         if (my $v = $drive->{"iops${dir}"}) {
1840             $opts .= ",throttling.iops$qmpname=$v";
1841         }
1842         if (my $v = $drive->{"iops${dir}_max"}) {
1843             $opts .= ",throttling.iops$qmpname-max=$v";
1844         }
1845         if (my $v = $drive->{"iops${dir}_max_length"}) {
1846             $opts .= ",throttling.iops$qmpname-max-length=$v";
1847         }
1848     }
1849
1850     $opts .= ",format=$format" if $format && !$drive->{format};
1851
1852     my $cache_direct = 0;
1853
1854     if (my $cache = $drive->{cache}) {
1855         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1856     } elsif (!drive_is_cdrom($drive)) {
1857         $opts .= ",cache=none";
1858         $cache_direct = 1;
1859     }
1860
1861     # aio native works only with O_DIRECT
1862     if (!$drive->{aio}) {
1863         if($cache_direct) {
1864             $opts .= ",aio=native";
1865         } else {
1866             $opts .= ",aio=threads";
1867         }
1868     }
1869
1870     if (!drive_is_cdrom($drive)) {
1871         my $detectzeroes;
1872         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1873             $detectzeroes = 'off';
1874         } elsif ($drive->{discard}) {
1875             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1876         } else {
1877             # This used to be our default with discard not being specified:
1878             $detectzeroes = 'on';
1879         }
1880         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1881     }
1882
1883     my $pathinfo = $path ? "file=$path," : '';
1884
1885     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1886 }
1887
1888 sub print_netdevice_full {
1889     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1890
1891     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1892
1893     my $device = $net->{model};
1894     if ($net->{model} eq 'virtio') {
1895          $device = 'virtio-net-pci';
1896      };
1897
1898     my $pciaddr = print_pci_addr("$netid", $bridges);
1899     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1900     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1901         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1902         my $vectors = $net->{queues} * 2 + 2;
1903         $tmpstr .= ",vectors=$vectors,mq=on";
1904     }
1905     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1906
1907     if ($use_old_bios_files) {
1908         my $romfile;
1909         if ($device eq 'virtio-net-pci') {
1910             $romfile = 'pxe-virtio.rom';
1911         } elsif ($device eq 'e1000') {
1912             $romfile = 'pxe-e1000.rom';
1913         } elsif ($device eq 'ne2k') {
1914             $romfile = 'pxe-ne2k_pci.rom';
1915         } elsif ($device eq 'pcnet') {
1916             $romfile = 'pxe-pcnet.rom';
1917         } elsif ($device eq 'rtl8139') {
1918             $romfile = 'pxe-rtl8139.rom';
1919         }
1920         $tmpstr .= ",romfile=$romfile" if $romfile;
1921     }
1922
1923     return $tmpstr;
1924 }
1925
1926 sub print_netdev_full {
1927     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1928
1929     my $i = '';
1930     if ($netid =~ m/^net(\d+)$/) {
1931         $i = int($1);
1932     }
1933
1934     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1935
1936     my $ifname = "tap${vmid}i$i";
1937
1938     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1939     die "interface name '$ifname' is too long (max 15 character)\n"
1940         if length($ifname) >= 16;
1941
1942     my $vhostparam = '';
1943     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1944
1945     my $vmname = $conf->{name} || "vm$vmid";
1946
1947     my $netdev = "";
1948     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1949
1950     if ($net->{bridge}) {
1951         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1952     } else {
1953         $netdev = "type=user,id=$netid,hostname=$vmname";
1954     }
1955
1956     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1957
1958     return $netdev;
1959 }
1960
1961
1962 sub print_cpu_device {
1963     my ($conf, $id) = @_;
1964
1965     my $kvm = $conf->{kvm} // 1;
1966     my $cpu = $kvm ? "kvm64" : "qemu64";
1967     if (my $cputype = $conf->{cpu}) {
1968         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1969             or die "Cannot parse cpu description: $cputype\n";
1970         $cpu = $cpuconf->{cputype};
1971     }
1972
1973     my $cores = $conf->{cores} || 1;
1974
1975     my $current_core = ($id - 1) % $cores;
1976     my $current_socket = int(($id - 1 - $current_core)/$cores);
1977
1978     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1979 }
1980
1981 my $vga_map = {
1982     'cirrus' => 'cirrus-vga',
1983     'std' => 'VGA',
1984     'vmware' => 'vmware-svga',
1985     'virtio' => 'virtio-vga',
1986 };
1987
1988 sub print_vga_device {
1989     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
1990
1991     my $type = $vga_map->{$vga->{type}};
1992     my $vgamem_mb = $vga->{memory};
1993     if ($qxlnum) {
1994         $type = $id ? 'qxl' : 'qxl-vga';
1995     }
1996     die "no devicetype for $vga->{type}\n" if !$type;
1997
1998     my $memory = "";
1999     if ($vgamem_mb) {
2000         if ($vga->{type} eq 'virtio') {
2001             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2002             $memory = ",max_hostmem=$bytes";
2003         } elsif ($qxlnum) {
2004             # from https://www.spice-space.org/multiple-monitors.html
2005             $memory = ",vgamem_mb=$vga->{memory}";
2006             my $ram = $vgamem_mb * 4;
2007             my $vram = $vgamem_mb * 2;
2008             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2009         } else {
2010             $memory = ",vgamem_mb=$vga->{memory}";
2011         }
2012     } elsif ($qxlnum && $id) {
2013         $memory = ",ram_size=67108864,vram_size=33554432";
2014     }
2015
2016     my $q35 = machine_type_is_q35($conf);
2017     my $vgaid = "vga" . ($id // '');
2018     my $pciaddr;
2019     if ($q35 && $vgaid eq 'vga') {
2020         # on is on the pcie.0 bus on q35
2021         $pciaddr = print_pcie_addr($vgaid, $bridges);
2022     } else {
2023         $pciaddr = print_pci_addr($vgaid, $bridges);
2024     }
2025
2026     return "$type,id=${vgaid}${memory}${pciaddr}";
2027 }
2028
2029 sub drive_is_cloudinit {
2030     my ($drive) = @_;
2031     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2032 }
2033
2034 sub drive_is_cdrom {
2035     my ($drive, $exclude_cloudinit) = @_;
2036
2037     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2038
2039     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2040
2041 }
2042
2043 sub parse_number_sets {
2044     my ($set) = @_;
2045     my $res = [];
2046     foreach my $part (split(/;/, $set)) {
2047         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2048             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2049             push @$res, [ $1, $2 ];
2050         } else {
2051             die "invalid range: $part\n";
2052         }
2053     }
2054     return $res;
2055 }
2056
2057 sub parse_numa {
2058     my ($data) = @_;
2059
2060     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2061     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2062     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2063     return $res;
2064 }
2065
2066 sub parse_hostpci {
2067     my ($value) = @_;
2068
2069     return undef if !$value;
2070
2071     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2072
2073     my @idlist = split(/;/, $res->{host});
2074     delete $res->{host};
2075     foreach my $id (@idlist) {
2076         if ($id =~ /^$PCIRE$/) {
2077             if (defined($2)) {
2078                 push @{$res->{pciid}}, { id => $1, function => $2 };
2079             } else {
2080                 my $pcidevices = lspci($1);
2081                 $res->{pciid} = $pcidevices->{$1};
2082             }
2083         } else {
2084             # should have been caught by parse_property_string already
2085             die "failed to parse PCI id: $id\n";
2086         }
2087     }
2088     return $res;
2089 }
2090
2091 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2092 sub parse_net {
2093     my ($data) = @_;
2094
2095     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2096     if ($@) {
2097         warn $@;
2098         return undef;
2099     }
2100     if (!defined($res->{macaddr})) {
2101         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2102         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2103     }
2104     return $res;
2105 }
2106
2107 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2108 sub parse_ipconfig {
2109     my ($data) = @_;
2110
2111     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2112     if ($@) {
2113         warn $@;
2114         return undef;
2115     }
2116
2117     if ($res->{gw} && !$res->{ip}) {
2118         warn 'gateway specified without specifying an IP address';
2119         return undef;
2120     }
2121     if ($res->{gw6} && !$res->{ip6}) {
2122         warn 'IPv6 gateway specified without specifying an IPv6 address';
2123         return undef;
2124     }
2125     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2126         warn 'gateway specified together with DHCP';
2127         return undef;
2128     }
2129     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2130         # gw6 + auto/dhcp
2131         warn "IPv6 gateway specified together with $res->{ip6} address";
2132         return undef;
2133     }
2134
2135     if (!$res->{ip} && !$res->{ip6}) {
2136         return { ip => 'dhcp', ip6 => 'dhcp' };
2137     }
2138
2139     return $res;
2140 }
2141
2142 sub print_net {
2143     my $net = shift;
2144
2145     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2146 }
2147
2148 sub add_random_macs {
2149     my ($settings) = @_;
2150
2151     foreach my $opt (keys %$settings) {
2152         next if $opt !~ m/^net(\d+)$/;
2153         my $net = parse_net($settings->{$opt});
2154         next if !$net;
2155         $settings->{$opt} = print_net($net);
2156     }
2157 }
2158
2159 sub vm_is_volid_owner {
2160     my ($storecfg, $vmid, $volid) = @_;
2161
2162     if ($volid !~  m|^/|) {
2163         my ($path, $owner);
2164         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2165         if ($owner && ($owner == $vmid)) {
2166             return 1;
2167         }
2168     }
2169
2170     return undef;
2171 }
2172
2173 sub split_flagged_list {
2174     my $text = shift || '';
2175     $text =~ s/[,;]/ /g;
2176     $text =~ s/^\s+//;
2177     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2178 }
2179
2180 sub join_flagged_list {
2181     my ($how, $lst) = @_;
2182     join $how, map { $lst->{$_} . $_ } keys %$lst;
2183 }
2184
2185 sub vmconfig_delete_pending_option {
2186     my ($conf, $key, $force) = @_;
2187
2188     delete $conf->{pending}->{$key};
2189     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2190     $pending_delete_hash->{$key} = $force ? '!' : '';
2191     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2192 }
2193
2194 sub vmconfig_undelete_pending_option {
2195     my ($conf, $key) = @_;
2196
2197     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2198     delete $pending_delete_hash->{$key};
2199
2200     if (%$pending_delete_hash) {
2201         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2202     } else {
2203         delete $conf->{pending}->{delete};
2204     }
2205 }
2206
2207 sub vmconfig_register_unused_drive {
2208     my ($storecfg, $vmid, $conf, $drive) = @_;
2209
2210     if (drive_is_cloudinit($drive)) {
2211         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2212         warn $@ if $@;
2213     } elsif (!drive_is_cdrom($drive)) {
2214         my $volid = $drive->{file};
2215         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2216             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2217         }
2218     }
2219 }
2220
2221 sub vmconfig_cleanup_pending {
2222     my ($conf) = @_;
2223
2224     # remove pending changes when nothing changed
2225     my $changes;
2226     foreach my $opt (keys %{$conf->{pending}}) {
2227         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2228             $changes = 1;
2229             delete $conf->{pending}->{$opt};
2230         }
2231     }
2232
2233     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2234     my $pending_delete_hash = {};
2235     while (my ($opt, $force) = each %$current_delete_hash) {
2236         if (defined($conf->{$opt})) {
2237             $pending_delete_hash->{$opt} = $force;
2238         } else {
2239             $changes = 1;
2240         }
2241     }
2242
2243     if (%$pending_delete_hash) {
2244         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2245     } else {
2246         delete $conf->{pending}->{delete};
2247     }
2248
2249     return $changes;
2250 }
2251
2252 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2253 my $smbios1_fmt = {
2254     uuid => {
2255         type => 'string',
2256         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2257         format_description => 'UUID',
2258         description => "Set SMBIOS1 UUID.",
2259         optional => 1,
2260     },
2261     version => {
2262         type => 'string',
2263         pattern => '\S+',
2264         format_description => 'string',
2265         description => "Set SMBIOS1 version.",
2266         optional => 1,
2267     },
2268     serial => {
2269         type => 'string',
2270         pattern => '\S+',
2271         format_description => 'string',
2272         description => "Set SMBIOS1 serial number.",
2273         optional => 1,
2274     },
2275     manufacturer => {
2276         type => 'string',
2277         pattern => '\S+',
2278         format_description => 'string',
2279         description => "Set SMBIOS1 manufacturer.",
2280         optional => 1,
2281     },
2282     product => {
2283         type => 'string',
2284         pattern => '\S+',
2285         format_description => 'string',
2286         description => "Set SMBIOS1 product ID.",
2287         optional => 1,
2288     },
2289     sku => {
2290         type => 'string',
2291         pattern => '\S+',
2292         format_description => 'string',
2293         description => "Set SMBIOS1 SKU string.",
2294         optional => 1,
2295     },
2296     family => {
2297         type => 'string',
2298         pattern => '\S+',
2299         format_description => 'string',
2300         description => "Set SMBIOS1 family string.",
2301         optional => 1,
2302     },
2303 };
2304
2305 sub parse_smbios1 {
2306     my ($data) = @_;
2307
2308     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2309     warn $@ if $@;
2310     return $res;
2311 }
2312
2313 sub print_smbios1 {
2314     my ($smbios1) = @_;
2315     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2316 }
2317
2318 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2319
2320 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2321 sub verify_bootdisk {
2322     my ($value, $noerr) = @_;
2323
2324     return $value if is_valid_drivename($value);
2325
2326     return undef if $noerr;
2327
2328     die "invalid boot disk '$value'\n";
2329 }
2330
2331 sub parse_watchdog {
2332     my ($value) = @_;
2333
2334     return undef if !$value;
2335
2336     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2337     warn $@ if $@;
2338     return $res;
2339 }
2340
2341 sub parse_guest_agent {
2342     my ($value) = @_;
2343
2344     return {} if !defined($value->{agent});
2345
2346     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2347     warn $@ if $@;
2348
2349     # if the agent is disabled ignore the other potentially set properties
2350     return {} if !$res->{enabled};
2351     return $res;
2352 }
2353
2354 sub parse_vga {
2355     my ($value) = @_;
2356
2357     return {} if !$value;
2358     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2359     warn $@ if $@;
2360     return $res;
2361 }
2362
2363 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2364 sub verify_usb_device {
2365     my ($value, $noerr) = @_;
2366
2367     return $value if parse_usb_device($value);
2368
2369     return undef if $noerr;
2370
2371     die "unable to parse usb device\n";
2372 }
2373
2374 # add JSON properties for create and set function
2375 sub json_config_properties {
2376     my $prop = shift;
2377
2378     foreach my $opt (keys %$confdesc) {
2379         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2380         $prop->{$opt} = $confdesc->{$opt};
2381     }
2382
2383     return $prop;
2384 }
2385
2386 # return copy of $confdesc_cloudinit to generate documentation
2387 sub cloudinit_config_properties {
2388
2389     return dclone($confdesc_cloudinit);
2390 }
2391
2392 sub check_type {
2393     my ($key, $value) = @_;
2394
2395     die "unknown setting '$key'\n" if !$confdesc->{$key};
2396
2397     my $type = $confdesc->{$key}->{type};
2398
2399     if (!defined($value)) {
2400         die "got undefined value\n";
2401     }
2402
2403     if ($value =~ m/[\n\r]/) {
2404         die "property contains a line feed\n";
2405     }
2406
2407     if ($type eq 'boolean') {
2408         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2409         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2410         die "type check ('boolean') failed - got '$value'\n";
2411     } elsif ($type eq 'integer') {
2412         return int($1) if $value =~ m/^(\d+)$/;
2413         die "type check ('integer') failed - got '$value'\n";
2414     } elsif ($type eq 'number') {
2415         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2416         die "type check ('number') failed - got '$value'\n";
2417     } elsif ($type eq 'string') {
2418         if (my $fmt = $confdesc->{$key}->{format}) {
2419             PVE::JSONSchema::check_format($fmt, $value);
2420             return $value;
2421         }
2422         $value =~ s/^\"(.*)\"$/$1/;
2423         return $value;
2424     } else {
2425         die "internal error"
2426     }
2427 }
2428
2429 sub check_iommu_support{
2430     #fixme : need to check IOMMU support
2431     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2432
2433     my $iommu=1;
2434     return $iommu;
2435
2436 }
2437
2438 sub touch_config {
2439     my ($vmid) = @_;
2440
2441     my $conf = PVE::QemuConfig->config_file($vmid);
2442     utime undef, undef, $conf;
2443 }
2444
2445 sub destroy_vm {
2446     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2447
2448     my $conffile = PVE::QemuConfig->config_file($vmid);
2449
2450     my $conf = PVE::QemuConfig->load_config($vmid);
2451
2452     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2453
2454     if ($conf->{template}) {
2455         # check if any base image is still used by a linked clone
2456         foreach_drive($conf, sub {
2457                 my ($ds, $drive) = @_;
2458
2459                 return if drive_is_cdrom($drive);
2460
2461                 my $volid = $drive->{file};
2462
2463                 return if !$volid || $volid =~ m|^/|;
2464
2465                 die "base volume '$volid' is still in use by linked cloned\n"
2466                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2467
2468         });
2469     }
2470
2471     # only remove disks owned by this VM
2472     foreach_drive($conf, sub {
2473         my ($ds, $drive) = @_;
2474
2475         return if drive_is_cdrom($drive, 1);
2476
2477         my $volid = $drive->{file};
2478
2479         return if !$volid || $volid =~ m|^/|;
2480
2481         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2482         return if !$path || !$owner || ($owner != $vmid);
2483
2484         eval {
2485             PVE::Storage::vdisk_free($storecfg, $volid);
2486         };
2487         warn "Could not remove disk '$volid', check manually: $@" if $@;
2488
2489     });
2490
2491     if ($keep_empty_config) {
2492         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2493     } else {
2494         unlink $conffile;
2495     }
2496
2497     # also remove unused disk
2498     eval {
2499         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2500
2501         eval {
2502             PVE::Storage::foreach_volid($dl, sub {
2503                 my ($volid, $sid, $volname, $d) = @_;
2504                 PVE::Storage::vdisk_free($storecfg, $volid);
2505             });
2506         };
2507         warn $@ if $@;
2508
2509     };
2510     warn $@ if $@;
2511 }
2512
2513 sub parse_vm_config {
2514     my ($filename, $raw) = @_;
2515
2516     return undef if !defined($raw);
2517
2518     my $res = {
2519         digest => Digest::SHA::sha1_hex($raw),
2520         snapshots => {},
2521         pending => {},
2522     };
2523
2524     $filename =~ m|/qemu-server/(\d+)\.conf$|
2525         || die "got strange filename '$filename'";
2526
2527     my $vmid = $1;
2528
2529     my $conf = $res;
2530     my $descr;
2531     my $section = '';
2532
2533     my @lines = split(/\n/, $raw);
2534     foreach my $line (@lines) {
2535         next if $line =~ m/^\s*$/;
2536
2537         if ($line =~ m/^\[PENDING\]\s*$/i) {
2538             $section = 'pending';
2539             if (defined($descr)) {
2540                 $descr =~ s/\s+$//;
2541                 $conf->{description} = $descr;
2542             }
2543             $descr = undef;
2544             $conf = $res->{$section} = {};
2545             next;
2546
2547         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2548             $section = $1;
2549             if (defined($descr)) {
2550                 $descr =~ s/\s+$//;
2551                 $conf->{description} = $descr;
2552             }
2553             $descr = undef;
2554             $conf = $res->{snapshots}->{$section} = {};
2555             next;
2556         }
2557
2558         if ($line =~ m/^\#(.*)\s*$/) {
2559             $descr = '' if !defined($descr);
2560             $descr .= PVE::Tools::decode_text($1) . "\n";
2561             next;
2562         }
2563
2564         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2565             $descr = '' if !defined($descr);
2566             $descr .= PVE::Tools::decode_text($2);
2567         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2568             $conf->{snapstate} = $1;
2569         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2570             my $key = $1;
2571             my $value = $2;
2572             $conf->{$key} = $value;
2573         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2574             my $value = $1;
2575             if ($section eq 'pending') {
2576                 $conf->{delete} = $value; # we parse this later
2577             } else {
2578                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2579             }
2580         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2581             my $key = $1;
2582             my $value = $2;
2583             eval { $value = check_type($key, $value); };
2584             if ($@) {
2585                 warn "vm $vmid - unable to parse value of '$key' - $@";
2586             } else {
2587                 $key = 'ide2' if $key eq 'cdrom';
2588                 my $fmt = $confdesc->{$key}->{format};
2589                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2590                     my $v = parse_drive($key, $value);
2591                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2592                         $v->{file} = $volid;
2593                         $value = print_drive($vmid, $v);
2594                     } else {
2595                         warn "vm $vmid - unable to parse value of '$key'\n";
2596                         next;
2597                     }
2598                 }
2599
2600                 $conf->{$key} = $value;
2601             }
2602         }
2603     }
2604
2605     if (defined($descr)) {
2606         $descr =~ s/\s+$//;
2607         $conf->{description} = $descr;
2608     }
2609     delete $res->{snapstate}; # just to be sure
2610
2611     return $res;
2612 }
2613
2614 sub write_vm_config {
2615     my ($filename, $conf) = @_;
2616
2617     delete $conf->{snapstate}; # just to be sure
2618
2619     if ($conf->{cdrom}) {
2620         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2621         $conf->{ide2} = $conf->{cdrom};
2622         delete $conf->{cdrom};
2623     }
2624
2625     # we do not use 'smp' any longer
2626     if ($conf->{sockets}) {
2627         delete $conf->{smp};
2628     } elsif ($conf->{smp}) {
2629         $conf->{sockets} = $conf->{smp};
2630         delete $conf->{cores};
2631         delete $conf->{smp};
2632     }
2633
2634     my $used_volids = {};
2635
2636     my $cleanup_config = sub {
2637         my ($cref, $pending, $snapname) = @_;
2638
2639         foreach my $key (keys %$cref) {
2640             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2641                 $key eq 'snapstate' || $key eq 'pending';
2642             my $value = $cref->{$key};
2643             if ($key eq 'delete') {
2644                 die "propertry 'delete' is only allowed in [PENDING]\n"
2645                     if !$pending;
2646                 # fixme: check syntax?
2647                 next;
2648             }
2649             eval { $value = check_type($key, $value); };
2650             die "unable to parse value of '$key' - $@" if $@;
2651
2652             $cref->{$key} = $value;
2653
2654             if (!$snapname && is_valid_drivename($key)) {
2655                 my $drive = parse_drive($key, $value);
2656                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2657             }
2658         }
2659     };
2660
2661     &$cleanup_config($conf);
2662
2663     &$cleanup_config($conf->{pending}, 1);
2664
2665     foreach my $snapname (keys %{$conf->{snapshots}}) {
2666         die "internal error" if $snapname eq 'pending';
2667         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2668     }
2669
2670     # remove 'unusedX' settings if we re-add a volume
2671     foreach my $key (keys %$conf) {
2672         my $value = $conf->{$key};
2673         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2674             delete $conf->{$key};
2675         }
2676     }
2677
2678     my $generate_raw_config = sub {
2679         my ($conf, $pending) = @_;
2680
2681         my $raw = '';
2682
2683         # add description as comment to top of file
2684         if (defined(my $descr = $conf->{description})) {
2685             if ($descr) {
2686                 foreach my $cl (split(/\n/, $descr)) {
2687                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2688                 }
2689             } else {
2690                 $raw .= "#\n" if $pending;
2691             }
2692         }
2693
2694         foreach my $key (sort keys %$conf) {
2695             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2696             $raw .= "$key: $conf->{$key}\n";
2697         }
2698         return $raw;
2699     };
2700
2701     my $raw = &$generate_raw_config($conf);
2702
2703     if (scalar(keys %{$conf->{pending}})){
2704         $raw .= "\n[PENDING]\n";
2705         $raw .= &$generate_raw_config($conf->{pending}, 1);
2706     }
2707
2708     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2709         $raw .= "\n[$snapname]\n";
2710         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2711     }
2712
2713     return $raw;
2714 }
2715
2716 sub load_defaults {
2717
2718     my $res = {};
2719
2720     # we use static defaults from our JSON schema configuration
2721     foreach my $key (keys %$confdesc) {
2722         if (defined(my $default = $confdesc->{$key}->{default})) {
2723             $res->{$key} = $default;
2724         }
2725     }
2726
2727     return $res;
2728 }
2729
2730 sub config_list {
2731     my $vmlist = PVE::Cluster::get_vmlist();
2732     my $res = {};
2733     return $res if !$vmlist || !$vmlist->{ids};
2734     my $ids = $vmlist->{ids};
2735
2736     foreach my $vmid (keys %$ids) {
2737         my $d = $ids->{$vmid};
2738         next if !$d->{node} || $d->{node} ne $nodename;
2739         next if !$d->{type} || $d->{type} ne 'qemu';
2740         $res->{$vmid}->{exists} = 1;
2741     }
2742     return $res;
2743 }
2744
2745 # test if VM uses local resources (to prevent migration)
2746 sub check_local_resources {
2747     my ($conf, $noerr) = @_;
2748
2749     my $loc_res = 0;
2750
2751     $loc_res = 1 if $conf->{hostusb}; # old syntax
2752     $loc_res = 1 if $conf->{hostpci}; # old syntax
2753
2754     foreach my $k (keys %$conf) {
2755         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2756         # sockets are safe: they will recreated be on the target side post-migrate
2757         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2758         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2759     }
2760
2761     die "VM uses local resources\n" if $loc_res && !$noerr;
2762
2763     return $loc_res;
2764 }
2765
2766 # check if used storages are available on all nodes (use by migrate)
2767 sub check_storage_availability {
2768     my ($storecfg, $conf, $node) = @_;
2769
2770     foreach_drive($conf, sub {
2771         my ($ds, $drive) = @_;
2772
2773         my $volid = $drive->{file};
2774         return if !$volid;
2775
2776         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2777         return if !$sid;
2778
2779         # check if storage is available on both nodes
2780         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2781         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2782    });
2783 }
2784
2785 # list nodes where all VM images are available (used by has_feature API)
2786 sub shared_nodes {
2787     my ($conf, $storecfg) = @_;
2788
2789     my $nodelist = PVE::Cluster::get_nodelist();
2790     my $nodehash = { map { $_ => 1 } @$nodelist };
2791     my $nodename = PVE::INotify::nodename();
2792
2793     foreach_drive($conf, sub {
2794         my ($ds, $drive) = @_;
2795
2796         my $volid = $drive->{file};
2797         return if !$volid;
2798
2799         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2800         if ($storeid) {
2801             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2802             if ($scfg->{disable}) {
2803                 $nodehash = {};
2804             } elsif (my $avail = $scfg->{nodes}) {
2805                 foreach my $node (keys %$nodehash) {
2806                     delete $nodehash->{$node} if !$avail->{$node};
2807                 }
2808             } elsif (!$scfg->{shared}) {
2809                 foreach my $node (keys %$nodehash) {
2810                     delete $nodehash->{$node} if $node ne $nodename
2811                 }
2812             }
2813         }
2814     });
2815
2816     return $nodehash
2817 }
2818
2819 sub check_cmdline {
2820     my ($pidfile, $pid) = @_;
2821
2822     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2823     if (defined($fh)) {
2824         my $line = <$fh>;
2825         $fh->close;
2826         return undef if !$line;
2827         my @param = split(/\0/, $line);
2828
2829         my $cmd = $param[0];
2830         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2831
2832         for (my $i = 0; $i < scalar (@param); $i++) {
2833             my $p = $param[$i];
2834             next if !$p;
2835             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2836                 my $p = $param[$i+1];
2837                 return 1 if $p && ($p eq $pidfile);
2838                 return undef;
2839             }
2840         }
2841     }
2842     return undef;
2843 }
2844
2845 sub check_running {
2846     my ($vmid, $nocheck, $node) = @_;
2847
2848     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2849
2850     die "unable to find configuration file for VM $vmid - no such machine\n"
2851         if !$nocheck && ! -f $filename;
2852
2853     my $pidfile = pidfile_name($vmid);
2854
2855     if (my $fd = IO::File->new("<$pidfile")) {
2856         my $st = stat($fd);
2857         my $line = <$fd>;
2858         close($fd);
2859
2860         my $mtime = $st->mtime;
2861         if ($mtime > time()) {
2862             warn "file '$filename' modified in future\n";
2863         }
2864
2865         if ($line =~ m/^(\d+)$/) {
2866             my $pid = $1;
2867             if (check_cmdline($pidfile, $pid)) {
2868                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2869                     return $pid;
2870                 }
2871             }
2872         }
2873     }
2874
2875     return undef;
2876 }
2877
2878 sub vzlist {
2879
2880     my $vzlist = config_list();
2881
2882     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2883
2884     while (defined(my $de = $fd->read)) {
2885         next if $de !~ m/^(\d+)\.pid$/;
2886         my $vmid = $1;
2887         next if !defined($vzlist->{$vmid});
2888         if (my $pid = check_running($vmid)) {
2889             $vzlist->{$vmid}->{pid} = $pid;
2890         }
2891     }
2892
2893     return $vzlist;
2894 }
2895
2896 sub disksize {
2897     my ($storecfg, $conf) = @_;
2898
2899     my $bootdisk = $conf->{bootdisk};
2900     return undef if !$bootdisk;
2901     return undef if !is_valid_drivename($bootdisk);
2902
2903     return undef if !$conf->{$bootdisk};
2904
2905     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2906     return undef if !defined($drive);
2907
2908     return undef if drive_is_cdrom($drive);
2909
2910     my $volid = $drive->{file};
2911     return undef if !$volid;
2912
2913     return $drive->{size};
2914 }
2915
2916 our $vmstatus_return_properties = {
2917     vmid => get_standard_option('pve-vmid'),
2918     status => {
2919         description => "Qemu process status.",
2920         type => 'string',
2921         enum => ['stopped', 'running'],
2922     },
2923     maxmem => {
2924         description => "Maximum memory in bytes.",
2925         type => 'integer',
2926         optional => 1,
2927         renderer => 'bytes',
2928     },
2929     maxdisk => {
2930         description => "Root disk size in bytes.",
2931         type => 'integer',
2932         optional => 1,
2933         renderer => 'bytes',
2934     },
2935     name => {
2936         description => "VM name.",
2937         type => 'string',
2938         optional => 1,
2939     },
2940     qmpstatus => {
2941         description => "Qemu QMP agent status.",
2942         type => 'string',
2943         optional => 1,
2944     },
2945     pid => {
2946         description => "PID of running qemu process.",
2947         type => 'integer',
2948         optional => 1,
2949     },
2950     uptime => {
2951         description => "Uptime.",
2952         type => 'integer',
2953         optional => 1,
2954         renderer => 'duration',
2955     },
2956     cpus => {
2957         description => "Maximum usable CPUs.",
2958         type => 'number',
2959         optional => 1,
2960     },
2961 };
2962
2963 my $last_proc_pid_stat;
2964
2965 # get VM status information
2966 # This must be fast and should not block ($full == false)
2967 # We only query KVM using QMP if $full == true (this can be slow)
2968 sub vmstatus {
2969     my ($opt_vmid, $full) = @_;
2970
2971     my $res = {};
2972
2973     my $storecfg = PVE::Storage::config();
2974
2975     my $list = vzlist();
2976     my $defaults = load_defaults();
2977
2978     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2979
2980     my $cpucount = $cpuinfo->{cpus} || 1;
2981
2982     foreach my $vmid (keys %$list) {
2983         next if $opt_vmid && ($vmid ne $opt_vmid);
2984
2985         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2986         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2987
2988         my $d = { vmid => $vmid };
2989         $d->{pid} = $list->{$vmid}->{pid};
2990
2991         # fixme: better status?
2992         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2993
2994         my $size = disksize($storecfg, $conf);
2995         if (defined($size)) {
2996             $d->{disk} = 0; # no info available
2997             $d->{maxdisk} = $size;
2998         } else {
2999             $d->{disk} = 0;
3000             $d->{maxdisk} = 0;
3001         }
3002
3003         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3004             * ($conf->{cores} || $defaults->{cores});
3005         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3006         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3007
3008         $d->{name} = $conf->{name} || "VM $vmid";
3009         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3010             : $defaults->{memory}*(1024*1024);
3011
3012         if ($conf->{balloon}) {
3013             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3014             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3015                 : $defaults->{shares};
3016         }
3017
3018         $d->{uptime} = 0;
3019         $d->{cpu} = 0;
3020         $d->{mem} = 0;
3021
3022         $d->{netout} = 0;
3023         $d->{netin} = 0;
3024
3025         $d->{diskread} = 0;
3026         $d->{diskwrite} = 0;
3027
3028         $d->{template} = PVE::QemuConfig->is_template($conf);
3029
3030         $d->{serial} = 1 if conf_has_serial($conf);
3031
3032         $res->{$vmid} = $d;
3033     }
3034
3035     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3036     foreach my $dev (keys %$netdev) {
3037         next if $dev !~ m/^tap([1-9]\d*)i/;
3038         my $vmid = $1;
3039         my $d = $res->{$vmid};
3040         next if !$d;
3041
3042         $d->{netout} += $netdev->{$dev}->{receive};
3043         $d->{netin} += $netdev->{$dev}->{transmit};
3044
3045         if ($full) {
3046             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3047             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3048         }
3049
3050     }
3051
3052     my $ctime = gettimeofday;
3053
3054     foreach my $vmid (keys %$list) {
3055
3056         my $d = $res->{$vmid};
3057         my $pid = $d->{pid};
3058         next if !$pid;
3059
3060         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3061         next if !$pstat; # not running
3062
3063         my $used = $pstat->{utime} + $pstat->{stime};
3064
3065         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3066
3067         if ($pstat->{vsize}) {
3068             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3069         }
3070
3071         my $old = $last_proc_pid_stat->{$pid};
3072         if (!$old) {
3073             $last_proc_pid_stat->{$pid} = {
3074                 time => $ctime,
3075                 used => $used,
3076                 cpu => 0,
3077             };
3078             next;
3079         }
3080
3081         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3082
3083         if ($dtime > 1000) {
3084             my $dutime = $used -  $old->{used};
3085
3086             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3087             $last_proc_pid_stat->{$pid} = {
3088                 time => $ctime,
3089                 used => $used,
3090                 cpu => $d->{cpu},
3091             };
3092         } else {
3093             $d->{cpu} = $old->{cpu};
3094         }
3095     }
3096
3097     return $res if !$full;
3098
3099     my $qmpclient = PVE::QMPClient->new();
3100
3101     my $ballooncb = sub {
3102         my ($vmid, $resp) = @_;
3103
3104         my $info = $resp->{'return'};
3105         return if !$info->{max_mem};
3106
3107         my $d = $res->{$vmid};
3108
3109         # use memory assigned to VM
3110         $d->{maxmem} = $info->{max_mem};
3111         $d->{balloon} = $info->{actual};
3112
3113         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3114             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3115             $d->{freemem} = $info->{free_mem};
3116         }
3117
3118         $d->{ballooninfo} = $info;
3119     };
3120
3121     my $blockstatscb = sub {
3122         my ($vmid, $resp) = @_;
3123         my $data = $resp->{'return'} || [];
3124         my $totalrdbytes = 0;
3125         my $totalwrbytes = 0;
3126
3127         for my $blockstat (@$data) {
3128             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3129             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3130
3131             $blockstat->{device} =~ s/drive-//;
3132             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3133         }
3134         $res->{$vmid}->{diskread} = $totalrdbytes;
3135         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3136     };
3137
3138     my $statuscb = sub {
3139         my ($vmid, $resp) = @_;
3140
3141         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3142         # this fails if ballon driver is not loaded, so this must be
3143         # the last commnand (following command are aborted if this fails).
3144         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3145
3146         my $status = 'unknown';
3147         if (!defined($status = $resp->{'return'}->{status})) {
3148             warn "unable to get VM status\n";
3149             return;
3150         }
3151
3152         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3153     };
3154
3155     foreach my $vmid (keys %$list) {
3156         next if $opt_vmid && ($vmid ne $opt_vmid);
3157         next if !$res->{$vmid}->{pid}; # not running
3158         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3159     }
3160
3161     $qmpclient->queue_execute(undef, 2);
3162
3163     foreach my $vmid (keys %$list) {
3164         next if $opt_vmid && ($vmid ne $opt_vmid);
3165         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3166     }
3167
3168     return $res;
3169 }
3170
3171 sub foreach_drive {
3172     my ($conf, $func, @param) = @_;
3173
3174     foreach my $ds (valid_drive_names()) {
3175         next if !defined($conf->{$ds});
3176
3177         my $drive = parse_drive($ds, $conf->{$ds});
3178         next if !$drive;
3179
3180         &$func($ds, $drive, @param);
3181     }
3182 }
3183
3184 sub foreach_volid {
3185     my ($conf, $func, @param) = @_;
3186
3187     my $volhash = {};
3188
3189     my $test_volid = sub {
3190         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3191
3192         return if !$volid;
3193
3194         $volhash->{$volid}->{cdrom} //= 1;
3195         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3196
3197         $volhash->{$volid}->{replicate} //= 0;
3198         $volhash->{$volid}->{replicate} = 1 if $replicate;
3199
3200         $volhash->{$volid}->{shared} //= 0;
3201         $volhash->{$volid}->{shared} = 1 if $shared;
3202
3203         $volhash->{$volid}->{referenced_in_config} //= 0;
3204         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3205
3206         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3207             if defined($snapname);
3208     };
3209
3210     foreach_drive($conf, sub {
3211         my ($ds, $drive) = @_;
3212         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3213     });
3214
3215     foreach my $snapname (keys %{$conf->{snapshots}}) {
3216         my $snap = $conf->{snapshots}->{$snapname};
3217         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3218         foreach_drive($snap, sub {
3219             my ($ds, $drive) = @_;
3220             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3221         });
3222     }
3223
3224     foreach my $volid (keys %$volhash) {
3225         &$func($volid, $volhash->{$volid}, @param);
3226     }
3227 }
3228
3229 sub conf_has_serial {
3230     my ($conf) = @_;
3231
3232     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3233         if ($conf->{"serial$i"}) {
3234             return 1;
3235         }
3236     }
3237
3238     return 0;
3239 }
3240
3241 sub vga_conf_has_spice {
3242     my ($vga) = @_;
3243
3244     my $vgaconf = parse_vga($vga);
3245     my $vgatype = $vgaconf->{type};
3246     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3247
3248     return $1 || 1;
3249 }
3250
3251 sub config_to_command {
3252     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3253
3254     my $cmd = [];
3255     my $globalFlags = [];
3256     my $machineFlags = [];
3257     my $rtcFlags = [];
3258     my $cpuFlags = [];
3259     my $devices = [];
3260     my $pciaddr = '';
3261     my $bridges = {};
3262     my $kvmver = kvm_user_version();
3263     my $vernum = 0; # unknown
3264     my $ostype = $conf->{ostype};
3265     my $winversion = windows_version($ostype);
3266     my $kvm = $conf->{kvm} // 1;
3267
3268     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3269
3270     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3271         $vernum = $1*1000000+$2*1000;
3272     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3273         $vernum = $1*1000000+$2*1000+$3;
3274     }
3275
3276     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3277
3278     my $have_ovz = -f '/proc/vz/vestat';
3279
3280     my $q35 = machine_type_is_q35($conf);
3281     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3282     my $machine_type = $forcemachine || $conf->{machine};
3283     my $use_old_bios_files = undef;
3284     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3285
3286     my $cpuunits = defined($conf->{cpuunits}) ?
3287             $conf->{cpuunits} : $defaults->{cpuunits};
3288
3289     push @$cmd, '/usr/bin/kvm';
3290
3291     push @$cmd, '-id', $vmid;
3292
3293     my $vmname = $conf->{name} || "vm$vmid";
3294
3295     push @$cmd, '-name', $vmname;
3296
3297     my $use_virtio = 0;
3298
3299     my $qmpsocket = qmp_socket($vmid);
3300     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3301     push @$cmd, '-mon', "chardev=qmp,mode=control";
3302
3303     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3304         my $eventsocket = qmp_socket($vmid, 0, 'event');
3305         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3306         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3307     }
3308
3309     push @$cmd, '-pidfile' , pidfile_name($vmid);
3310
3311     push @$cmd, '-daemonize';
3312
3313     if ($conf->{smbios1}) {
3314         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3315     }
3316
3317     if ($conf->{vmgenid}) {
3318         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3319     }
3320
3321     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3322         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3323
3324         my $path;
3325         my $format;
3326         if (my $efidisk = $conf->{efidisk0}) {
3327             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3328             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3329             $format = $d->{format};
3330             if ($storeid) {
3331                 $path = PVE::Storage::path($storecfg, $d->{file});
3332                 if (!defined($format)) {
3333                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3334                     $format = qemu_img_format($scfg, $volname);
3335                 }
3336             } else {
3337                 $path = $d->{file};
3338                 die "efidisk format must be specified\n"
3339                     if !defined($format);
3340             }
3341         } else {
3342             warn "no efidisk configured! Using temporary efivars disk.\n";
3343             $path = "/tmp/$vmid-ovmf.fd";
3344             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3345             $format = 'raw';
3346         }
3347
3348         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3349         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3350     }
3351
3352
3353     # add usb controllers
3354     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3355     push @$devices, @usbcontrollers if @usbcontrollers;
3356     my $vga = parse_vga($conf->{vga});
3357
3358     my $qxlnum = vga_conf_has_spice($conf->{vga});
3359     $vga->{type} = 'qxl' if $qxlnum;
3360
3361     if (!$vga->{type}) {
3362         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3363             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3364         } else {
3365             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3366         }
3367     }
3368
3369     # enable absolute mouse coordinates (needed by vnc)
3370     my $tablet;
3371     if (defined($conf->{tablet})) {
3372         $tablet = $conf->{tablet};
3373     } else {
3374         $tablet = $defaults->{tablet};
3375         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3376         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3377     }
3378
3379     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3380
3381     my $kvm_off = 0;
3382     my $gpu_passthrough;
3383
3384     # host pci devices
3385     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3386         my $d = parse_hostpci($conf->{"hostpci$i"});
3387         next if !$d;
3388
3389         my $pcie = $d->{pcie};
3390         if($pcie){
3391             die "q35 machine model is not enabled" if !$q35;
3392             $pciaddr = print_pcie_addr("hostpci$i");
3393         }else{
3394             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3395         }
3396
3397         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3398         my $romfile = $d->{romfile};
3399
3400         my $xvga = '';
3401         if ($d->{'x-vga'}) {
3402             $xvga = ',x-vga=on';
3403             $kvm_off = 1;
3404             $vga->{type} = 'none';
3405             $gpu_passthrough = 1;
3406
3407             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3408                 $xvga = "";
3409             }
3410         }
3411         my $pcidevices = $d->{pciid};
3412         my $multifunction = 1 if @$pcidevices > 1;
3413
3414         my $j=0;
3415         foreach my $pcidevice (@$pcidevices) {
3416
3417             my $id = "hostpci$i";
3418             $id .= ".$j" if $multifunction;
3419             my $addr = $pciaddr;
3420             $addr .= ".$j" if $multifunction;
3421             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3422
3423             if($j == 0){
3424                 $devicestr .= "$rombar$xvga";
3425                 $devicestr .= ",multifunction=on" if $multifunction;
3426                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3427             }
3428
3429             push @$devices, '-device', $devicestr;
3430             $j++;
3431         }
3432     }
3433
3434     # usb devices
3435     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3436     push @$devices, @usbdevices if @usbdevices;
3437     # serial devices
3438     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3439         if (my $path = $conf->{"serial$i"}) {
3440             if ($path eq 'socket') {
3441                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3442                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3443                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3444             } else {
3445                 die "no such serial device\n" if ! -c $path;
3446                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3447                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3448             }
3449         }
3450     }
3451
3452     # parallel devices
3453     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3454         if (my $path = $conf->{"parallel$i"}) {
3455             die "no such parallel device\n" if ! -c $path;
3456             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3457             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3458             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3459         }
3460     }
3461
3462
3463     my $sockets = 1;
3464     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3465     $sockets = $conf->{sockets} if  $conf->{sockets};
3466
3467     my $cores = $conf->{cores} || 1;
3468
3469     my $maxcpus = $sockets * $cores;
3470
3471     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3472
3473     my $allowed_vcpus = $cpuinfo->{cpus};
3474
3475     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3476         if ($allowed_vcpus < $maxcpus);
3477
3478     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3479
3480         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3481         for (my $i = 2; $i <= $vcpus; $i++)  {
3482             my $cpustr = print_cpu_device($conf,$i);
3483             push @$cmd, '-device', $cpustr;
3484         }
3485
3486     } else {
3487
3488         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3489     }
3490     push @$cmd, '-nodefaults';
3491
3492     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3493
3494     my $bootindex_hash = {};
3495     my $i = 1;
3496     foreach my $o (split(//, $bootorder)) {
3497         $bootindex_hash->{$o} = $i*100;
3498         $i++;
3499     }
3500
3501     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3502
3503     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3504
3505     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3506
3507     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga ne 'none'){
3508         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3509         my $socket = vnc_socket($vmid);
3510         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3511     } else {
3512         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3513         push @$cmd, '-nographic';
3514     }
3515
3516     # time drift fix
3517     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3518
3519     my $useLocaltime = $conf->{localtime};
3520
3521     if ($winversion >= 5) { # windows
3522         $useLocaltime = 1 if !defined($conf->{localtime});
3523
3524         # use time drift fix when acpi is enabled
3525         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3526             $tdf = 1 if !defined($conf->{tdf});
3527         }
3528     }
3529
3530     if ($winversion >= 6) {
3531         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3532         push @$cmd, '-no-hpet';
3533     }
3534
3535     push @$rtcFlags, 'driftfix=slew' if $tdf;
3536
3537     if (!$kvm) {
3538         push @$machineFlags, 'accel=tcg';
3539     }
3540
3541     if ($machine_type) {
3542         push @$machineFlags, "type=${machine_type}";
3543     }
3544
3545     if ($conf->{startdate}) {
3546         push @$rtcFlags, "base=$conf->{startdate}";
3547     } elsif ($useLocaltime) {
3548         push @$rtcFlags, 'base=localtime';
3549     }
3550
3551     my $cpu = $kvm ? "kvm64" : "qemu64";
3552     if (my $cputype = $conf->{cpu}) {
3553         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3554             or die "Cannot parse cpu description: $cputype\n";
3555         $cpu = $cpuconf->{cputype};
3556         $kvm_off = 1 if $cpuconf->{hidden};
3557
3558         if (defined(my $flags = $cpuconf->{flags})) {
3559             push @$cpuFlags, split(";", $flags);
3560         }
3561     }
3562
3563     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3564
3565     push @$cpuFlags , '-x2apic'
3566         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3567
3568     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3569
3570     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3571
3572     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3573
3574         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3575         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3576     }
3577
3578     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3579
3580     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3581
3582     push @$cpuFlags, 'kvm=off' if $kvm_off;
3583
3584     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3585         die "internal error"; # should not happen
3586
3587     push @$cpuFlags, "vendor=${cpu_vendor}"
3588         if $cpu_vendor ne 'default';
3589
3590     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3591
3592     push @$cmd, '-cpu', $cpu;
3593
3594     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3595
3596     push @$cmd, '-S' if $conf->{freeze};
3597
3598     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3599
3600     # enable sound
3601     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3602     #push @$cmd, '-soundhw', 'es1370';
3603     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3604
3605     if (parse_guest_agent($conf)->{enabled}) {
3606         my $qgasocket = qmp_socket($vmid, 1);
3607         my $pciaddr = print_pci_addr("qga0", $bridges);
3608         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3609         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3610         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3611     }
3612
3613     my $spice_port;
3614
3615     if ($qxlnum) {
3616         if ($qxlnum > 1) {
3617             if ($winversion){
3618                 for(my $i = 1; $i < $qxlnum; $i++){
3619                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3620                 }
3621             } else {
3622                 # assume other OS works like Linux
3623                 my ($ram, $vram) = ("134217728", "67108864");
3624                 if ($vga->{memory}) {
3625                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3626                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3627                 }
3628                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3629                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3630             }
3631         }
3632
3633         my $pciaddr = print_pci_addr("spice", $bridges);
3634
3635         my $nodename = PVE::INotify::nodename();
3636         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3637         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3638         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3639         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3640         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3641
3642         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3643
3644         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3645         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3646         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3647     }
3648
3649     # enable balloon by default, unless explicitly disabled
3650     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3651         $pciaddr = print_pci_addr("balloon0", $bridges);
3652         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3653     }
3654
3655     if ($conf->{watchdog}) {
3656         my $wdopts = parse_watchdog($conf->{watchdog});
3657         $pciaddr = print_pci_addr("watchdog", $bridges);
3658         my $watchdog = $wdopts->{model} || 'i6300esb';
3659         push @$devices, '-device', "$watchdog$pciaddr";
3660         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3661     }
3662
3663     my $vollist = [];
3664     my $scsicontroller = {};
3665     my $ahcicontroller = {};
3666     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3667
3668     # Add iscsi initiator name if available
3669     if (my $initiator = get_initiator_name()) {
3670         push @$devices, '-iscsi', "initiator-name=$initiator";
3671     }
3672
3673     foreach_drive($conf, sub {
3674         my ($ds, $drive) = @_;
3675
3676         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3677             push @$vollist, $drive->{file};
3678         }
3679
3680         # ignore efidisk here, already added in bios/fw handling code above
3681         return if $drive->{interface} eq 'efidisk';
3682
3683         $use_virtio = 1 if $ds =~ m/^virtio/;
3684
3685         if (drive_is_cdrom ($drive)) {
3686             if ($bootindex_hash->{d}) {
3687                 $drive->{bootindex} = $bootindex_hash->{d};
3688                 $bootindex_hash->{d} += 1;
3689             }
3690         } else {
3691             if ($bootindex_hash->{c}) {
3692                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3693                 $bootindex_hash->{c} += 1;
3694             }
3695         }
3696
3697         if($drive->{interface} eq 'virtio'){
3698            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3699         }
3700
3701         if ($drive->{interface} eq 'scsi') {
3702
3703             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3704
3705             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3706             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3707
3708             my $iothread = '';
3709             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3710                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3711                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3712             } elsif ($drive->{iothread}) {
3713                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3714             }
3715
3716             my $queues = '';
3717             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3718                 $queues = ",num_queues=$drive->{queues}";
3719             }
3720
3721             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3722             $scsicontroller->{$controller}=1;
3723         }
3724
3725         if ($drive->{interface} eq 'sata') {
3726            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3727            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3728            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3729            $ahcicontroller->{$controller}=1;
3730         }
3731
3732         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3733         push @$devices, '-drive',$drive_cmd;
3734         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3735     });
3736
3737     for (my $i = 0; $i < $MAX_NETS; $i++) {
3738          next if !$conf->{"net$i"};
3739          my $d = parse_net($conf->{"net$i"});
3740          next if !$d;
3741
3742          $use_virtio = 1 if $d->{model} eq 'virtio';
3743
3744          if ($bootindex_hash->{n}) {
3745             $d->{bootindex} = $bootindex_hash->{n};
3746             $bootindex_hash->{n} += 1;
3747          }
3748
3749          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3750          push @$devices, '-netdev', $netdevfull;
3751
3752          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3753          push @$devices, '-device', $netdevicefull;
3754     }
3755
3756     if (!$q35) {
3757         # add pci bridges
3758         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3759            $bridges->{1} = 1;
3760            $bridges->{2} = 1;
3761         }
3762
3763         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3764
3765         while (my ($k, $v) = each %$bridges) {
3766             $pciaddr = print_pci_addr("pci.$k");
3767             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3768         }
3769     }
3770
3771     # add custom args
3772     if ($conf->{args}) {
3773         my $aa = PVE::Tools::split_args($conf->{args});
3774         push @$cmd, @$aa;
3775     }
3776
3777     push @$cmd, @$devices;
3778     push @$cmd, '-rtc', join(',', @$rtcFlags)
3779         if scalar(@$rtcFlags);
3780     push @$cmd, '-machine', join(',', @$machineFlags)
3781         if scalar(@$machineFlags);
3782     push @$cmd, '-global', join(',', @$globalFlags)
3783         if scalar(@$globalFlags);
3784
3785     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3786 }
3787
3788 sub vnc_socket {
3789     my ($vmid) = @_;
3790     return "${var_run_tmpdir}/$vmid.vnc";
3791 }
3792
3793 sub spice_port {
3794     my ($vmid) = @_;
3795
3796     my $res = vm_mon_cmd($vmid, 'query-spice');
3797
3798     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3799 }
3800
3801 sub qmp_socket {
3802     my ($vmid, $qga, $name) = @_;
3803     my $sockettype = $qga ? 'qga' : 'qmp';
3804     my $ext = $name ? '-'.$name : '';
3805     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3806 }
3807
3808 sub pidfile_name {
3809     my ($vmid) = @_;
3810     return "${var_run_tmpdir}/$vmid.pid";
3811 }
3812
3813 sub vm_devices_list {
3814     my ($vmid) = @_;
3815
3816     my $res = vm_mon_cmd($vmid, 'query-pci');
3817     my $devices_to_check = [];
3818     my $devices = {};
3819     foreach my $pcibus (@$res) {
3820         push @$devices_to_check, @{$pcibus->{devices}},
3821     }
3822
3823     while (@$devices_to_check) {
3824         my $to_check = [];
3825         for my $d (@$devices_to_check) {
3826             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3827             next if !$d->{'pci_bridge'};
3828
3829             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3830             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3831         }
3832         $devices_to_check = $to_check;
3833     }
3834
3835     my $resblock = vm_mon_cmd($vmid, 'query-block');
3836     foreach my $block (@$resblock) {
3837         if($block->{device} =~ m/^drive-(\S+)/){
3838                 $devices->{$1} = 1;
3839         }
3840     }
3841
3842     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3843     foreach my $mice (@$resmice) {
3844         if ($mice->{name} eq 'QEMU HID Tablet') {
3845             $devices->{tablet} = 1;
3846             last;
3847         }
3848     }
3849
3850     # for usb devices there is no query-usb
3851     # but we can iterate over the entries in
3852     # qom-list path=/machine/peripheral
3853     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3854     foreach my $per (@$resperipheral) {
3855         if ($per->{name} =~ m/^usb\d+$/) {
3856             $devices->{$per->{name}} = 1;
3857         }
3858     }
3859
3860     return $devices;
3861 }
3862
3863 sub vm_deviceplug {
3864     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3865
3866     my $q35 = machine_type_is_q35($conf);
3867
3868     my $devices_list = vm_devices_list($vmid);
3869     return 1 if defined($devices_list->{$deviceid});
3870
3871     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3872
3873     if ($deviceid eq 'tablet') {
3874
3875         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3876
3877     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3878
3879         die "usb hotplug currently not reliable\n";
3880         # since we can't reliably hot unplug all added usb devices
3881         # and usb passthrough disables live migration
3882         # we disable usb hotplugging for now
3883         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3884
3885     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3886
3887         qemu_iothread_add($vmid, $deviceid, $device);
3888
3889         qemu_driveadd($storecfg, $vmid, $device);
3890         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3891
3892         qemu_deviceadd($vmid, $devicefull);
3893         eval { qemu_deviceaddverify($vmid, $deviceid); };
3894         if (my $err = $@) {
3895             eval { qemu_drivedel($vmid, $deviceid); };
3896             warn $@ if $@;
3897             die $err;
3898         }
3899
3900     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3901
3902
3903         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3904         my $pciaddr = print_pci_addr($deviceid);
3905         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3906
3907         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3908
3909         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3910             qemu_iothread_add($vmid, $deviceid, $device);
3911             $devicefull .= ",iothread=iothread-$deviceid";
3912         }
3913
3914         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3915             $devicefull .= ",num_queues=$device->{queues}";
3916         }
3917
3918         qemu_deviceadd($vmid, $devicefull);
3919         qemu_deviceaddverify($vmid, $deviceid);
3920
3921     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3922
3923         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3924         qemu_driveadd($storecfg, $vmid, $device);
3925
3926         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3927         eval { qemu_deviceadd($vmid, $devicefull); };
3928         if (my $err = $@) {
3929             eval { qemu_drivedel($vmid, $deviceid); };
3930             warn $@ if $@;
3931             die $err;
3932         }
3933
3934     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3935
3936         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3937
3938         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3939         my $use_old_bios_files = undef;
3940         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3941
3942         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3943         qemu_deviceadd($vmid, $netdevicefull);
3944         eval { qemu_deviceaddverify($vmid, $deviceid); };
3945         if (my $err = $@) {
3946             eval { qemu_netdevdel($vmid, $deviceid); };
3947             warn $@ if $@;
3948             die $err;
3949         }
3950
3951     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3952
3953         my $bridgeid = $2;
3954         my $pciaddr = print_pci_addr($deviceid);
3955         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3956
3957         qemu_deviceadd($vmid, $devicefull);
3958         qemu_deviceaddverify($vmid, $deviceid);
3959
3960     } else {
3961         die "can't hotplug device '$deviceid'\n";
3962     }
3963
3964     return 1;
3965 }
3966
3967 # fixme: this should raise exceptions on error!
3968 sub vm_deviceunplug {
3969     my ($vmid, $conf, $deviceid) = @_;
3970
3971     my $devices_list = vm_devices_list($vmid);
3972     return 1 if !defined($devices_list->{$deviceid});
3973
3974     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3975
3976     if ($deviceid eq 'tablet') {
3977
3978         qemu_devicedel($vmid, $deviceid);
3979
3980     } elsif ($deviceid =~ m/^usb\d+$/) {
3981
3982         die "usb hotplug currently not reliable\n";
3983         # when unplugging usb devices this way,
3984         # there may be remaining usb controllers/hubs
3985         # so we disable it for now
3986         qemu_devicedel($vmid, $deviceid);
3987         qemu_devicedelverify($vmid, $deviceid);
3988
3989     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3990
3991         qemu_devicedel($vmid, $deviceid);
3992         qemu_devicedelverify($vmid, $deviceid);
3993         qemu_drivedel($vmid, $deviceid);
3994         qemu_iothread_del($conf, $vmid, $deviceid);
3995
3996     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3997
3998         qemu_devicedel($vmid, $deviceid);
3999         qemu_devicedelverify($vmid, $deviceid);
4000         qemu_iothread_del($conf, $vmid, $deviceid);
4001
4002     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4003
4004         qemu_devicedel($vmid, $deviceid);
4005         qemu_drivedel($vmid, $deviceid);
4006         qemu_deletescsihw($conf, $vmid, $deviceid);
4007
4008     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4009
4010         qemu_devicedel($vmid, $deviceid);
4011         qemu_devicedelverify($vmid, $deviceid);
4012         qemu_netdevdel($vmid, $deviceid);
4013
4014     } else {
4015         die "can't unplug device '$deviceid'\n";
4016     }
4017
4018     return 1;
4019 }
4020
4021 sub qemu_deviceadd {
4022     my ($vmid, $devicefull) = @_;
4023
4024     $devicefull = "driver=".$devicefull;
4025     my %options =  split(/[=,]/, $devicefull);
4026
4027     vm_mon_cmd($vmid, "device_add" , %options);
4028 }
4029
4030 sub qemu_devicedel {
4031     my ($vmid, $deviceid) = @_;
4032
4033     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4034 }
4035
4036 sub qemu_iothread_add {
4037     my($vmid, $deviceid, $device) = @_;
4038
4039     if ($device->{iothread}) {
4040         my $iothreads = vm_iothreads_list($vmid);
4041         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4042     }
4043 }
4044
4045 sub qemu_iothread_del {
4046     my($conf, $vmid, $deviceid) = @_;
4047
4048     my $device = parse_drive($deviceid, $conf->{$deviceid});
4049     if ($device->{iothread}) {
4050         my $iothreads = vm_iothreads_list($vmid);
4051         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4052     }
4053 }
4054
4055 sub qemu_objectadd {
4056     my($vmid, $objectid, $qomtype) = @_;
4057
4058     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4059
4060     return 1;
4061 }
4062
4063 sub qemu_objectdel {
4064     my($vmid, $objectid) = @_;
4065
4066     vm_mon_cmd($vmid, "object-del", id => $objectid);
4067
4068     return 1;
4069 }
4070
4071 sub qemu_driveadd {
4072     my ($storecfg, $vmid, $device) = @_;
4073
4074     my $drive = print_drive_full($storecfg, $vmid, $device);
4075     $drive =~ s/\\/\\\\/g;
4076     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4077
4078     # If the command succeeds qemu prints: "OK"
4079     return 1 if $ret =~ m/OK/s;
4080
4081     die "adding drive failed: $ret\n";
4082 }
4083
4084 sub qemu_drivedel {
4085     my($vmid, $deviceid) = @_;
4086
4087     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4088     $ret =~ s/^\s+//;
4089
4090     return 1 if $ret eq "";
4091
4092     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4093     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4094
4095     die "deleting drive $deviceid failed : $ret\n";
4096 }
4097
4098 sub qemu_deviceaddverify {
4099     my ($vmid, $deviceid) = @_;
4100
4101     for (my $i = 0; $i <= 5; $i++) {
4102          my $devices_list = vm_devices_list($vmid);
4103          return 1 if defined($devices_list->{$deviceid});
4104          sleep 1;
4105     }
4106
4107     die "error on hotplug device '$deviceid'\n";
4108 }
4109
4110
4111 sub qemu_devicedelverify {
4112     my ($vmid, $deviceid) = @_;
4113
4114     # need to verify that the device is correctly removed as device_del
4115     # is async and empty return is not reliable
4116
4117     for (my $i = 0; $i <= 5; $i++) {
4118          my $devices_list = vm_devices_list($vmid);
4119          return 1 if !defined($devices_list->{$deviceid});
4120          sleep 1;
4121     }
4122
4123     die "error on hot-unplugging device '$deviceid'\n";
4124 }
4125
4126 sub qemu_findorcreatescsihw {
4127     my ($storecfg, $conf, $vmid, $device) = @_;
4128
4129     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4130
4131     my $scsihwid="$controller_prefix$controller";
4132     my $devices_list = vm_devices_list($vmid);
4133
4134     if(!defined($devices_list->{$scsihwid})) {
4135         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4136     }
4137
4138     return 1;
4139 }
4140
4141 sub qemu_deletescsihw {
4142     my ($conf, $vmid, $opt) = @_;
4143
4144     my $device = parse_drive($opt, $conf->{$opt});
4145
4146     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4147         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4148         return 1;
4149     }
4150
4151     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4152
4153     my $devices_list = vm_devices_list($vmid);
4154     foreach my $opt (keys %{$devices_list}) {
4155         if (PVE::QemuServer::is_valid_drivename($opt)) {
4156             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4157             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4158                 return 1;
4159             }
4160         }
4161     }
4162
4163     my $scsihwid="scsihw$controller";
4164
4165     vm_deviceunplug($vmid, $conf, $scsihwid);
4166
4167     return 1;
4168 }
4169
4170 sub qemu_add_pci_bridge {
4171     my ($storecfg, $conf, $vmid, $device) = @_;
4172
4173     my $bridges = {};
4174
4175     my $bridgeid;
4176
4177     print_pci_addr($device, $bridges);
4178
4179     while (my ($k, $v) = each %$bridges) {
4180         $bridgeid = $k;
4181     }
4182     return 1 if !defined($bridgeid) || $bridgeid < 1;
4183
4184     my $bridge = "pci.$bridgeid";
4185     my $devices_list = vm_devices_list($vmid);
4186
4187     if (!defined($devices_list->{$bridge})) {
4188         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4189     }
4190
4191     return 1;
4192 }
4193
4194 sub qemu_set_link_status {
4195     my ($vmid, $device, $up) = @_;
4196
4197     vm_mon_cmd($vmid, "set_link", name => $device,
4198                up => $up ? JSON::true : JSON::false);
4199 }
4200
4201 sub qemu_netdevadd {
4202     my ($vmid, $conf, $device, $deviceid) = @_;
4203
4204     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4205     my %options =  split(/[=,]/, $netdev);
4206
4207     vm_mon_cmd($vmid, "netdev_add",  %options);
4208     return 1;
4209 }
4210
4211 sub qemu_netdevdel {
4212     my ($vmid, $deviceid) = @_;
4213
4214     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4215 }
4216
4217 sub qemu_usb_hotplug {
4218     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4219
4220     return if !$device;
4221
4222     # remove the old one first
4223     vm_deviceunplug($vmid, $conf, $deviceid);
4224
4225     # check if xhci controller is necessary and available
4226     if ($device->{usb3}) {
4227
4228         my $devicelist = vm_devices_list($vmid);
4229
4230         if (!$devicelist->{xhci}) {
4231             my $pciaddr = print_pci_addr("xhci");
4232             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4233         }
4234     }
4235     my $d = parse_usb_device($device->{host});
4236     $d->{usb3} = $device->{usb3};
4237
4238     # add the new one
4239     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4240 }
4241
4242 sub qemu_cpu_hotplug {
4243     my ($vmid, $conf, $vcpus) = @_;
4244
4245     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4246
4247     my $sockets = 1;
4248     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4249     $sockets = $conf->{sockets} if  $conf->{sockets};
4250     my $cores = $conf->{cores} || 1;
4251     my $maxcpus = $sockets * $cores;
4252
4253     $vcpus = $maxcpus if !$vcpus;
4254
4255     die "you can't add more vcpus than maxcpus\n"
4256         if $vcpus > $maxcpus;
4257
4258     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4259
4260     if ($vcpus < $currentvcpus) {
4261
4262         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4263
4264             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4265                 qemu_devicedel($vmid, "cpu$i");
4266                 my $retry = 0;
4267                 my $currentrunningvcpus = undef;
4268                 while (1) {
4269                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4270                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4271                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4272                     $retry++;
4273                     sleep 1;
4274                 }
4275                 #update conf after each succesfull cpu unplug
4276                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4277                 PVE::QemuConfig->write_config($vmid, $conf);
4278             }
4279         } else {
4280             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4281         }
4282
4283         return;
4284     }
4285
4286     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4287     die "vcpus in running vm does not match its configuration\n"
4288         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4289
4290     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4291
4292         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4293             my $cpustr = print_cpu_device($conf, $i);
4294             qemu_deviceadd($vmid, $cpustr);
4295
4296             my $retry = 0;
4297             my $currentrunningvcpus = undef;
4298             while (1) {
4299                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4300                 last if scalar(@{$currentrunningvcpus}) == $i;
4301                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4302                 sleep 1;
4303                 $retry++;
4304             }
4305             #update conf after each succesfull cpu hotplug
4306             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4307             PVE::QemuConfig->write_config($vmid, $conf);
4308         }
4309     } else {
4310
4311         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4312             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4313         }
4314     }
4315 }
4316
4317 sub qemu_block_set_io_throttle {
4318     my ($vmid, $deviceid,
4319         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4320         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4321         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4322         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4323
4324     return if !check_running($vmid) ;
4325
4326     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4327         bps => int($bps),
4328         bps_rd => int($bps_rd),
4329         bps_wr => int($bps_wr),
4330         iops => int($iops),
4331         iops_rd => int($iops_rd),
4332         iops_wr => int($iops_wr),
4333         bps_max => int($bps_max),
4334         bps_rd_max => int($bps_rd_max),
4335         bps_wr_max => int($bps_wr_max),
4336         iops_max => int($iops_max),
4337         iops_rd_max => int($iops_rd_max),
4338         iops_wr_max => int($iops_wr_max),
4339         bps_max_length => int($bps_max_length),
4340         bps_rd_max_length => int($bps_rd_max_length),
4341         bps_wr_max_length => int($bps_wr_max_length),
4342         iops_max_length => int($iops_max_length),
4343         iops_rd_max_length => int($iops_rd_max_length),
4344         iops_wr_max_length => int($iops_wr_max_length),
4345     );
4346
4347 }
4348
4349 # old code, only used to shutdown old VM after update
4350 sub __read_avail {
4351     my ($fh, $timeout) = @_;
4352
4353     my $sel = new IO::Select;
4354     $sel->add($fh);
4355
4356     my $res = '';
4357     my $buf;
4358
4359     my @ready;
4360     while (scalar (@ready = $sel->can_read($timeout))) {
4361         my $count;
4362         if ($count = $fh->sysread($buf, 8192)) {
4363             if ($buf =~ /^(.*)\(qemu\) $/s) {
4364                 $res .= $1;
4365                 last;
4366             } else {
4367                 $res .= $buf;
4368             }
4369         } else {
4370             if (!defined($count)) {
4371                 die "$!\n";
4372             }
4373             last;
4374         }
4375     }
4376
4377     die "monitor read timeout\n" if !scalar(@ready);
4378
4379     return $res;
4380 }
4381
4382 sub qemu_block_resize {
4383     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4384
4385     my $running = check_running($vmid);
4386
4387     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4388
4389     return if !$running;
4390
4391     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4392
4393 }
4394
4395 sub qemu_volume_snapshot {
4396     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4397
4398     my $running = check_running($vmid);
4399
4400     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4401         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4402     } else {
4403         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4404     }
4405 }
4406
4407 sub qemu_volume_snapshot_delete {
4408     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4409
4410     my $running = check_running($vmid);
4411
4412     if($running) {
4413
4414         $running = undef;
4415         my $conf = PVE::QemuConfig->load_config($vmid);
4416         foreach_drive($conf, sub {
4417             my ($ds, $drive) = @_;
4418             $running = 1 if $drive->{file} eq $volid;
4419         });
4420     }
4421
4422     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4423         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4424     } else {
4425         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4426     }
4427 }
4428
4429 sub set_migration_caps {
4430     my ($vmid) = @_;
4431
4432     my $cap_ref = [];
4433
4434     my $enabled_cap = {
4435         "auto-converge" => 1,
4436         "xbzrle" => 1,
4437         "x-rdma-pin-all" => 0,
4438         "zero-blocks" => 0,
4439         "compress" => 0
4440     };
4441
4442     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4443
4444     for my $supported_capability (@$supported_capabilities) {
4445         push @$cap_ref, {
4446             capability => $supported_capability->{capability},
4447             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4448         };
4449     }
4450
4451     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4452 }
4453
4454 my $fast_plug_option = {
4455     'lock' => 1,
4456     'name' => 1,
4457     'onboot' => 1,
4458     'shares' => 1,
4459     'startup' => 1,
4460     'description' => 1,
4461     'protection' => 1,
4462     'vmstatestorage' => 1,
4463 };
4464
4465 # hotplug changes in [PENDING]
4466 # $selection hash can be used to only apply specified options, for
4467 # example: { cores => 1 } (only apply changed 'cores')
4468 # $errors ref is used to return error messages
4469 sub vmconfig_hotplug_pending {
4470     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4471
4472     my $defaults = load_defaults();
4473
4474     # commit values which do not have any impact on running VM first
4475     # Note: those option cannot raise errors, we we do not care about
4476     # $selection and always apply them.
4477
4478     my $add_error = sub {
4479         my ($opt, $msg) = @_;
4480         $errors->{$opt} = "hotplug problem - $msg";
4481     };
4482
4483     my $changes = 0;
4484     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4485         if ($fast_plug_option->{$opt}) {
4486             $conf->{$opt} = $conf->{pending}->{$opt};
4487             delete $conf->{pending}->{$opt};
4488             $changes = 1;
4489         }
4490     }
4491
4492     if ($changes) {
4493         PVE::QemuConfig->write_config($vmid, $conf);
4494         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4495     }
4496
4497     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4498
4499     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4500     while (my ($opt, $force) = each %$pending_delete_hash) {
4501         next if $selection && !$selection->{$opt};
4502         eval {
4503             if ($opt eq 'hotplug') {
4504                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4505             } elsif ($opt eq 'tablet') {
4506                 die "skip\n" if !$hotplug_features->{usb};
4507                 if ($defaults->{tablet}) {
4508                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4509                 } else {
4510                     vm_deviceunplug($vmid, $conf, $opt);
4511                 }
4512             } elsif ($opt =~ m/^usb\d+/) {
4513                 die "skip\n";
4514                 # since we cannot reliably hot unplug usb devices
4515                 # we are disabling it
4516                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4517                 vm_deviceunplug($vmid, $conf, $opt);
4518             } elsif ($opt eq 'vcpus') {
4519                 die "skip\n" if !$hotplug_features->{cpu};
4520                 qemu_cpu_hotplug($vmid, $conf, undef);
4521             } elsif ($opt eq 'balloon') {
4522                 # enable balloon device is not hotpluggable
4523                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4524                 # here we reset the ballooning value to memory
4525                 my $balloon = $conf->{memory} || $defaults->{memory};
4526                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4527             } elsif ($fast_plug_option->{$opt}) {
4528                 # do nothing
4529             } elsif ($opt =~ m/^net(\d+)$/) {
4530                 die "skip\n" if !$hotplug_features->{network};
4531                 vm_deviceunplug($vmid, $conf, $opt);
4532             } elsif (is_valid_drivename($opt)) {
4533                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4534                 vm_deviceunplug($vmid, $conf, $opt);
4535                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4536             } elsif ($opt =~ m/^memory$/) {
4537                 die "skip\n" if !$hotplug_features->{memory};
4538                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4539             } elsif ($opt eq 'cpuunits') {
4540                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4541             } elsif ($opt eq 'cpulimit') {
4542                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4543             } else {
4544                 die "skip\n";
4545             }
4546         };
4547         if (my $err = $@) {
4548             &$add_error($opt, $err) if $err ne "skip\n";
4549         } else {
4550             # save new config if hotplug was successful
4551             delete $conf->{$opt};
4552             vmconfig_undelete_pending_option($conf, $opt);
4553             PVE::QemuConfig->write_config($vmid, $conf);
4554             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4555         }
4556     }
4557
4558     my $apply_pending_cloudinit;
4559     $apply_pending_cloudinit = sub {
4560         my ($key, $value) = @_;
4561         $apply_pending_cloudinit = sub {}; # once is enough
4562
4563         my @cloudinit_opts = keys %$confdesc_cloudinit;
4564         foreach my $opt (keys %{$conf->{pending}}) {
4565             next if !grep { $_ eq $opt } @cloudinit_opts;
4566             $conf->{$opt} = delete $conf->{pending}->{$opt};
4567         }
4568
4569         my $new_conf = { %$conf };
4570         $new_conf->{$key} = $value;
4571         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4572     };
4573
4574     foreach my $opt (keys %{$conf->{pending}}) {
4575         next if $selection && !$selection->{$opt};
4576         my $value = $conf->{pending}->{$opt};
4577         eval {
4578             if ($opt eq 'hotplug') {
4579                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4580             } elsif ($opt eq 'tablet') {
4581                 die "skip\n" if !$hotplug_features->{usb};
4582                 if ($value == 1) {
4583                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4584                 } elsif ($value == 0) {
4585                     vm_deviceunplug($vmid, $conf, $opt);
4586                 }
4587             } elsif ($opt =~ m/^usb\d+$/) {
4588                 die "skip\n";
4589                 # since we cannot reliably hot unplug usb devices
4590                 # we are disabling it
4591                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4592                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4593                 die "skip\n" if !$d;
4594                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4595             } elsif ($opt eq 'vcpus') {
4596                 die "skip\n" if !$hotplug_features->{cpu};
4597                 qemu_cpu_hotplug($vmid, $conf, $value);
4598             } elsif ($opt eq 'balloon') {
4599                 # enable/disable balloning device is not hotpluggable
4600                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4601                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4602                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4603
4604                 # allow manual ballooning if shares is set to zero
4605                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4606                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4607                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4608                 }
4609             } elsif ($opt =~ m/^net(\d+)$/) {
4610                 # some changes can be done without hotplug
4611                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4612                                     $vmid, $opt, $value);
4613             } elsif (is_valid_drivename($opt)) {
4614                 # some changes can be done without hotplug
4615                 my $drive = parse_drive($opt, $value);
4616                 if (drive_is_cloudinit($drive)) {
4617                     &$apply_pending_cloudinit($opt, $value);
4618                 }
4619                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4620                                      $vmid, $opt, $value, 1);
4621             } elsif ($opt =~ m/^memory$/) { #dimms
4622                 die "skip\n" if !$hotplug_features->{memory};
4623                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4624             } elsif ($opt eq 'cpuunits') {
4625                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4626             } elsif ($opt eq 'cpulimit') {
4627                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4628                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4629             } else {
4630                 die "skip\n";  # skip non-hot-pluggable options
4631             }
4632         };
4633         if (my $err = $@) {
4634             &$add_error($opt, $err) if $err ne "skip\n";
4635         } else {
4636             # save new config if hotplug was successful
4637             $conf->{$opt} = $value;
4638             delete $conf->{pending}->{$opt};
4639             PVE::QemuConfig->write_config($vmid, $conf);
4640             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4641         }
4642     }
4643 }
4644
4645 sub try_deallocate_drive {
4646     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4647
4648     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4649         my $volid = $drive->{file};
4650         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4651             my $sid = PVE::Storage::parse_volume_id($volid);
4652             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4653
4654             # check if the disk is really unused
4655             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4656                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4657             PVE::Storage::vdisk_free($storecfg, $volid);
4658             return 1;
4659         } else {
4660             # If vm is not owner of this disk remove from config
4661             return 1;
4662         }
4663     }
4664
4665     return undef;
4666 }
4667
4668 sub vmconfig_delete_or_detach_drive {
4669     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4670
4671     my $drive = parse_drive($opt, $conf->{$opt});
4672
4673     my $rpcenv = PVE::RPCEnvironment::get();
4674     my $authuser = $rpcenv->get_user();
4675
4676     if ($force) {
4677         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4678         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4679     } else {
4680         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4681     }
4682 }
4683
4684 sub vmconfig_apply_pending {
4685     my ($vmid, $conf, $storecfg) = @_;
4686
4687     # cold plug
4688
4689     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4690     while (my ($opt, $force) = each %$pending_delete_hash) {
4691         die "internal error" if $opt =~ m/^unused/;
4692         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4693         if (!defined($conf->{$opt})) {
4694             vmconfig_undelete_pending_option($conf, $opt);
4695             PVE::QemuConfig->write_config($vmid, $conf);
4696         } elsif (is_valid_drivename($opt)) {
4697             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4698             vmconfig_undelete_pending_option($conf, $opt);
4699             delete $conf->{$opt};
4700             PVE::QemuConfig->write_config($vmid, $conf);
4701         } else {
4702             vmconfig_undelete_pending_option($conf, $opt);
4703             delete $conf->{$opt};
4704             PVE::QemuConfig->write_config($vmid, $conf);
4705         }
4706     }
4707
4708     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4709
4710     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4711         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4712
4713         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4714             # skip if nothing changed
4715         } elsif (is_valid_drivename($opt)) {
4716             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4717                 if defined($conf->{$opt});
4718             $conf->{$opt} = $conf->{pending}->{$opt};
4719         } else {
4720             $conf->{$opt} = $conf->{pending}->{$opt};
4721         }
4722
4723         delete $conf->{pending}->{$opt};
4724         PVE::QemuConfig->write_config($vmid, $conf);
4725     }
4726 }
4727
4728 my $safe_num_ne = sub {
4729     my ($a, $b) = @_;
4730
4731     return 0 if !defined($a) && !defined($b);
4732     return 1 if !defined($a);
4733     return 1 if !defined($b);
4734
4735     return $a != $b;
4736 };
4737
4738 my $safe_string_ne = sub {
4739     my ($a, $b) = @_;
4740
4741     return 0 if !defined($a) && !defined($b);
4742     return 1 if !defined($a);
4743     return 1 if !defined($b);
4744
4745     return $a ne $b;
4746 };
4747
4748 sub vmconfig_update_net {
4749     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4750
4751     my $newnet = parse_net($value);
4752
4753     if ($conf->{$opt}) {
4754         my $oldnet = parse_net($conf->{$opt});
4755
4756         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4757             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4758             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4759             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4760
4761             # for non online change, we try to hot-unplug
4762             die "skip\n" if !$hotplug;
4763             vm_deviceunplug($vmid, $conf, $opt);
4764         } else {
4765
4766             die "internal error" if $opt !~ m/net(\d+)/;
4767             my $iface = "tap${vmid}i$1";
4768
4769             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4770                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4771                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4772                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4773                 PVE::Network::tap_unplug($iface);
4774                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4775             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4776                 # Rate can be applied on its own but any change above needs to
4777                 # include the rate in tap_plug since OVS resets everything.
4778                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4779             }
4780
4781             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4782                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4783             }
4784
4785             return 1;
4786         }
4787     }
4788
4789     if ($hotplug) {
4790         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4791     } else {
4792         die "skip\n";
4793     }
4794 }
4795
4796 sub vmconfig_update_disk {
4797     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4798
4799     # fixme: do we need force?
4800
4801     my $drive = parse_drive($opt, $value);
4802
4803     if ($conf->{$opt}) {
4804
4805         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4806
4807             my $media = $drive->{media} || 'disk';
4808             my $oldmedia = $old_drive->{media} || 'disk';
4809             die "unable to change media type\n" if $media ne $oldmedia;
4810
4811             if (!drive_is_cdrom($old_drive)) {
4812
4813                 if ($drive->{file} ne $old_drive->{file}) {
4814
4815                     die "skip\n" if !$hotplug;
4816
4817                     # unplug and register as unused
4818                     vm_deviceunplug($vmid, $conf, $opt);
4819                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4820
4821                 } else {
4822                     # update existing disk
4823
4824                     # skip non hotpluggable value
4825                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4826                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4827                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4828                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4829                         die "skip\n";
4830                     }
4831
4832                     # apply throttle
4833                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4834                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4835                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4836                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4837                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4838                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4839                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4840                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4841                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4842                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4843                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4844                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4845                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4846                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4847                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4848                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4849                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4850                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4851
4852                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4853                                                    ($drive->{mbps} || 0)*1024*1024,
4854                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4855                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4856                                                    $drive->{iops} || 0,
4857                                                    $drive->{iops_rd} || 0,
4858                                                    $drive->{iops_wr} || 0,
4859                                                    ($drive->{mbps_max} || 0)*1024*1024,
4860                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4861                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4862                                                    $drive->{iops_max} || 0,
4863                                                    $drive->{iops_rd_max} || 0,
4864                                                    $drive->{iops_wr_max} || 0,
4865                                                    $drive->{bps_max_length} || 1,
4866                                                    $drive->{bps_rd_max_length} || 1,
4867                                                    $drive->{bps_wr_max_length} || 1,
4868                                                    $drive->{iops_max_length} || 1,
4869                                                    $drive->{iops_rd_max_length} || 1,
4870                                                    $drive->{iops_wr_max_length} || 1);
4871
4872                     }
4873
4874                     return 1;
4875                 }
4876
4877             } else { # cdrom
4878
4879                 if ($drive->{file} eq 'none') {
4880                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4881                     if (drive_is_cloudinit($old_drive)) {
4882                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4883                     }
4884                 } else {
4885                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4886                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4887                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4888                 }
4889
4890                 return 1;
4891             }
4892         }
4893     }
4894
4895     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4896     # hotplug new disks
4897     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4898     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4899 }
4900
4901 sub vm_start {
4902     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4903         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4904
4905     PVE::QemuConfig->lock_config($vmid, sub {
4906         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4907
4908         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4909
4910         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4911
4912         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4913
4914         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4915             vmconfig_apply_pending($vmid, $conf, $storecfg);
4916             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4917         }
4918
4919         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4920
4921         my $defaults = load_defaults();
4922
4923         # set environment variable useful inside network script
4924         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4925
4926         my $local_volumes = {};
4927
4928         if ($targetstorage) {
4929             foreach_drive($conf, sub {
4930                 my ($ds, $drive) = @_;
4931
4932                 return if drive_is_cdrom($drive);
4933
4934                 my $volid = $drive->{file};
4935
4936                 return if !$volid;
4937
4938                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4939
4940                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4941                 return if $scfg->{shared};
4942                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4943             });
4944
4945             my $format = undef;
4946
4947             foreach my $opt (sort keys %$local_volumes) {
4948
4949                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4950                 my $drive = parse_drive($opt, $conf->{$opt});
4951
4952                 #if remote storage is specified, use default format
4953                 if ($targetstorage && $targetstorage ne "1") {
4954                     $storeid = $targetstorage;
4955                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4956                     $format = $defFormat;
4957                 } else {
4958                     #else we use same format than original
4959                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4960                     $format = qemu_img_format($scfg, $volid);
4961                 }
4962
4963                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4964                 my $newdrive = $drive;
4965                 $newdrive->{format} = $format;
4966                 $newdrive->{file} = $newvolid;
4967                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4968                 $local_volumes->{$opt} = $drivestr;
4969                 #pass drive to conf for command line
4970                 $conf->{$opt} = $drivestr;
4971             }
4972         }
4973
4974         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4975
4976         my $migrate_port = 0;
4977         my $migrate_uri;
4978         if ($statefile) {
4979             if ($statefile eq 'tcp') {
4980                 my $localip = "localhost";
4981                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4982                 my $nodename = PVE::INotify::nodename();
4983
4984                 if (!defined($migration_type)) {
4985                     if (defined($datacenterconf->{migration}->{type})) {
4986                         $migration_type = $datacenterconf->{migration}->{type};
4987                     } else {
4988                         $migration_type = 'secure';
4989                     }
4990                 }
4991
4992                 if ($migration_type eq 'insecure') {
4993                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4994                     if ($migrate_network_addr) {
4995                         $localip = $migrate_network_addr;
4996                     } else {
4997                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4998                     }
4999
5000                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5001                 }
5002
5003                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5004                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5005                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5006                 push @$cmd, '-incoming', $migrate_uri;
5007                 push @$cmd, '-S';
5008
5009             } elsif ($statefile eq 'unix') {
5010                 # should be default for secure migrations as a ssh TCP forward
5011                 # tunnel is not deterministic reliable ready and fails regurarly
5012                 # to set up in time, so use UNIX socket forwards
5013                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5014                 unlink $socket_addr;
5015
5016                 $migrate_uri = "unix:$socket_addr";
5017
5018                 push @$cmd, '-incoming', $migrate_uri;
5019                 push @$cmd, '-S';
5020
5021             } else {
5022                 push @$cmd, '-loadstate', $statefile;
5023             }
5024         } elsif ($paused) {
5025             push @$cmd, '-S';
5026         }
5027
5028         # host pci devices
5029         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5030           my $d = parse_hostpci($conf->{"hostpci$i"});
5031           next if !$d;
5032           my $pcidevices = $d->{pciid};
5033           foreach my $pcidevice (@$pcidevices) {
5034                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5035
5036                 my $info = pci_device_info("0000:$pciid");
5037                 die "IOMMU not present\n" if !check_iommu_support();
5038                 die "no pci device info for device '$pciid'\n" if !$info;
5039                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5040                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5041           }
5042         }
5043
5044         PVE::Storage::activate_volumes($storecfg, $vollist);
5045
5046         if (!check_running($vmid, 1)) {
5047             eval {
5048                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5049                     outfunc => sub {}, errfunc => sub {});
5050             };
5051         }
5052
5053         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5054                                                   : $defaults->{cpuunits};
5055
5056         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5057         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5058
5059         my %properties = (
5060             Slice => 'qemu.slice',
5061             KillMode => 'none',
5062             CPUShares => $cpuunits
5063         );
5064
5065         if (my $cpulimit = $conf->{cpulimit}) {
5066             $properties{CPUQuota} = int($cpulimit * 100);
5067         }
5068         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5069
5070         my $run_qemu = sub {
5071             PVE::Tools::run_fork sub {
5072                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5073                 run_command($cmd, %run_params);
5074             };
5075         };
5076
5077         if ($conf->{hugepages}) {
5078
5079             my $code = sub {
5080                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5081                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5082
5083                 PVE::QemuServer::Memory::hugepages_mount();
5084                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5085
5086                 eval { $run_qemu->() };
5087                 if (my $err = $@) {
5088                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5089                     die $err;
5090                 }
5091
5092                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5093             };
5094             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5095
5096         } else {
5097             eval { $run_qemu->() };
5098         }
5099
5100         if (my $err = $@) {
5101             # deactivate volumes if start fails
5102             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5103             die "start failed: $err";
5104         }
5105
5106         print "migration listens on $migrate_uri\n" if $migrate_uri;
5107
5108         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5109             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5110             warn $@ if $@;
5111         }
5112
5113         #start nbd server for storage migration
5114         if ($targetstorage) {
5115             my $nodename = PVE::INotify::nodename();
5116             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5117             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5118             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5119             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5120
5121             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5122
5123             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5124
5125             foreach my $opt (sort keys %$local_volumes) {
5126                 my $volid = $local_volumes->{$opt};
5127                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5128                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5129                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5130             }
5131         }
5132
5133         if ($migratedfrom) {
5134             eval {
5135                 set_migration_caps($vmid);
5136             };
5137             warn $@ if $@;
5138
5139             if ($spice_port) {
5140                 print "spice listens on port $spice_port\n";
5141                 if ($spice_ticket) {
5142                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5143                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5144                 }
5145             }
5146
5147         } else {
5148             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5149                 if !$statefile && $conf->{balloon};
5150
5151             foreach my $opt (keys %$conf) {
5152                 next if $opt !~  m/^net\d+$/;
5153                 my $nicconf = parse_net($conf->{$opt});
5154                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5155             }
5156         }
5157
5158         vm_mon_cmd_nocheck($vmid, 'qom-set',
5159                     path => "machine/peripheral/balloon0",
5160                     property => "guest-stats-polling-interval",
5161                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5162
5163     });
5164 }
5165
5166 sub vm_mon_cmd {
5167     my ($vmid, $execute, %params) = @_;
5168
5169     my $cmd = { execute => $execute, arguments => \%params };
5170     vm_qmp_command($vmid, $cmd);
5171 }
5172
5173 sub vm_mon_cmd_nocheck {
5174     my ($vmid, $execute, %params) = @_;
5175
5176     my $cmd = { execute => $execute, arguments => \%params };
5177     vm_qmp_command($vmid, $cmd, 1);
5178 }
5179
5180 sub vm_qmp_command {
5181     my ($vmid, $cmd, $nocheck) = @_;
5182
5183     my $res;
5184
5185     my $timeout;
5186     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5187         $timeout = $cmd->{arguments}->{timeout};
5188         delete $cmd->{arguments}->{timeout};
5189     }
5190
5191     eval {
5192         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5193         my $sname = qmp_socket($vmid);
5194         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5195             my $qmpclient = PVE::QMPClient->new();
5196
5197             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5198         } else {
5199             die "unable to open monitor socket\n";
5200         }
5201     };
5202     if (my $err = $@) {
5203         syslog("err", "VM $vmid qmp command failed - $err");
5204         die $err;
5205     }
5206
5207     return $res;
5208 }
5209
5210 sub vm_human_monitor_command {
5211     my ($vmid, $cmdline) = @_;
5212
5213     my $res;
5214
5215     my $cmd = {
5216         execute => 'human-monitor-command',
5217         arguments => { 'command-line' => $cmdline},
5218     };
5219
5220     return vm_qmp_command($vmid, $cmd);
5221 }
5222
5223 sub vm_commandline {
5224     my ($storecfg, $vmid) = @_;
5225
5226     my $conf = PVE::QemuConfig->load_config($vmid);
5227
5228     my $defaults = load_defaults();
5229
5230     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5231
5232     return PVE::Tools::cmd2string($cmd);
5233 }
5234
5235 sub vm_reset {
5236     my ($vmid, $skiplock) = @_;
5237
5238     PVE::QemuConfig->lock_config($vmid, sub {
5239
5240         my $conf = PVE::QemuConfig->load_config($vmid);
5241
5242         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5243
5244         vm_mon_cmd($vmid, "system_reset");
5245     });
5246 }
5247
5248 sub get_vm_volumes {
5249     my ($conf) = @_;
5250
5251     my $vollist = [];
5252     foreach_volid($conf, sub {
5253         my ($volid, $attr) = @_;
5254
5255         return if $volid =~ m|^/|;
5256
5257         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5258         return if !$sid;
5259
5260         push @$vollist, $volid;
5261     });
5262
5263     return $vollist;
5264 }
5265
5266 sub vm_stop_cleanup {
5267     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5268
5269     eval {
5270
5271         if (!$keepActive) {
5272             my $vollist = get_vm_volumes($conf);
5273             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5274         }
5275
5276         foreach my $ext (qw(mon qmp pid vnc qga)) {
5277             unlink "/var/run/qemu-server/${vmid}.$ext";
5278         }
5279
5280         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5281     };
5282     warn $@ if $@; # avoid errors - just warn
5283 }
5284
5285 # Note: use $nockeck to skip tests if VM configuration file exists.
5286 # We need that when migration VMs to other nodes (files already moved)
5287 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5288 sub vm_stop {
5289     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5290
5291     $force = 1 if !defined($force) && !$shutdown;
5292
5293     if ($migratedfrom){
5294         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5295         kill 15, $pid if $pid;
5296         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5297         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5298         return;
5299     }
5300
5301     PVE::QemuConfig->lock_config($vmid, sub {
5302
5303         my $pid = check_running($vmid, $nocheck);
5304         return if !$pid;
5305
5306         my $conf;
5307         if (!$nocheck) {
5308             $conf = PVE::QemuConfig->load_config($vmid);
5309             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5310             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5311                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5312                 $timeout = $opts->{down} if $opts->{down};
5313             }
5314         }
5315
5316         $timeout = 60 if !defined($timeout);
5317
5318         eval {
5319             if ($shutdown) {
5320                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5321                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5322                 } else {
5323                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5324                 }
5325             } else {
5326                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5327             }
5328         };
5329         my $err = $@;
5330
5331         if (!$err) {
5332             my $count = 0;
5333             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5334                 $count++;
5335                 sleep 1;
5336             }
5337
5338             if ($count >= $timeout) {
5339                 if ($force) {
5340                     warn "VM still running - terminating now with SIGTERM\n";
5341                     kill 15, $pid;
5342                 } else {
5343                     die "VM quit/powerdown failed - got timeout\n";
5344                 }
5345             } else {
5346                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5347                 return;
5348             }
5349         } else {
5350             if ($force) {
5351                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5352                 kill 15, $pid;
5353             } else {
5354                 die "VM quit/powerdown failed\n";
5355             }
5356         }
5357
5358         # wait again
5359         $timeout = 10;
5360
5361         my $count = 0;
5362         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5363             $count++;
5364             sleep 1;
5365         }
5366
5367         if ($count >= $timeout) {
5368             warn "VM still running - terminating now with SIGKILL\n";
5369             kill 9, $pid;
5370             sleep 1;
5371         }
5372
5373         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5374    });
5375 }
5376
5377 sub vm_suspend {
5378     my ($vmid, $skiplock) = @_;
5379
5380     PVE::QemuConfig->lock_config($vmid, sub {
5381
5382         my $conf = PVE::QemuConfig->load_config($vmid);
5383
5384         PVE::QemuConfig->check_lock($conf)
5385             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5386
5387         vm_mon_cmd($vmid, "stop");
5388     });
5389 }
5390
5391 sub vm_resume {
5392     my ($vmid, $skiplock, $nocheck) = @_;
5393
5394     PVE::QemuConfig->lock_config($vmid, sub {
5395
5396         my $res = vm_mon_cmd($vmid, 'query-status');
5397         my $resume_cmd = 'cont';
5398
5399         if ($res->{status} && $res->{status} eq 'suspended') {
5400             $resume_cmd = 'system_wakeup';
5401         }
5402
5403         if (!$nocheck) {
5404
5405             my $conf = PVE::QemuConfig->load_config($vmid);
5406
5407             PVE::QemuConfig->check_lock($conf)
5408                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5409
5410             vm_mon_cmd($vmid, $resume_cmd);
5411
5412         } else {
5413             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5414         }
5415     });
5416 }
5417
5418 sub vm_sendkey {
5419     my ($vmid, $skiplock, $key) = @_;
5420
5421     PVE::QemuConfig->lock_config($vmid, sub {
5422
5423         my $conf = PVE::QemuConfig->load_config($vmid);
5424
5425         # there is no qmp command, so we use the human monitor command
5426         vm_human_monitor_command($vmid, "sendkey $key");
5427     });
5428 }
5429
5430 sub vm_destroy {
5431     my ($storecfg, $vmid, $skiplock) = @_;
5432
5433     PVE::QemuConfig->lock_config($vmid, sub {
5434
5435         my $conf = PVE::QemuConfig->load_config($vmid);
5436
5437         if (!check_running($vmid)) {
5438             destroy_vm($storecfg, $vmid, undef, $skiplock);
5439         } else {
5440             die "VM $vmid is running - destroy failed\n";
5441         }
5442     });
5443 }
5444
5445 # pci helpers
5446
5447 sub file_write {
5448     my ($filename, $buf) = @_;
5449
5450     my $fh = IO::File->new($filename, "w");
5451     return undef if !$fh;
5452
5453     my $res = print $fh $buf;
5454
5455     $fh->close();
5456
5457     return $res;
5458 }
5459
5460 sub pci_device_info {
5461     my ($name) = @_;
5462
5463     my $res;
5464
5465     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5466     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5467
5468     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5469     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5470
5471     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5472     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5473
5474     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5475     return undef if !defined($product) || $product !~ s/^0x//;
5476
5477     $res = {
5478         name => $name,
5479         vendor => $vendor,
5480         product => $product,
5481         domain => $domain,
5482         bus => $bus,
5483         slot => $slot,
5484         func => $func,
5485         irq => $irq,
5486         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5487     };
5488
5489     return $res;
5490 }
5491
5492 sub pci_dev_reset {
5493     my ($dev) = @_;
5494
5495     my $name = $dev->{name};
5496
5497     my $fn = "$pcisysfs/devices/$name/reset";
5498
5499     return file_write($fn, "1");
5500 }
5501
5502 sub pci_dev_bind_to_vfio {
5503     my ($dev) = @_;
5504
5505     my $name = $dev->{name};
5506
5507     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5508
5509     if (!-d $vfio_basedir) {
5510         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5511     }
5512     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5513
5514     my $testdir = "$vfio_basedir/$name";
5515     return 1 if -d $testdir;
5516
5517     my $data = "$dev->{vendor} $dev->{product}";
5518     return undef if !file_write("$vfio_basedir/new_id", $data);
5519
5520     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5521     if (!file_write($fn, $name)) {
5522         return undef if -f $fn;
5523     }
5524
5525     $fn = "$vfio_basedir/bind";
5526     if (! -d $testdir) {
5527         return undef if !file_write($fn, $name);
5528     }
5529
5530     return -d $testdir;
5531 }
5532
5533 sub pci_dev_group_bind_to_vfio {
5534     my ($pciid) = @_;
5535
5536     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5537
5538     if (!-d $vfio_basedir) {
5539         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5540     }
5541     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5542
5543     # get IOMMU group devices
5544     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5545       my @devs = grep /^0000:/, readdir($D);
5546     closedir($D);
5547
5548     foreach my $pciid (@devs) {
5549         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5550
5551         # pci bridges, switches or root ports are not supported
5552         # they have a pci_bus subdirectory so skip them
5553         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5554
5555         my $info = pci_device_info($1);
5556         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5557     }
5558
5559     return 1;
5560 }
5561
5562 # vzdump restore implementaion
5563
5564 sub tar_archive_read_firstfile {
5565     my $archive = shift;
5566
5567     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5568
5569     # try to detect archive type first
5570     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5571         die "unable to open file '$archive'\n";
5572     my $firstfile = <$fh>;
5573     kill 15, $pid;
5574     close $fh;
5575
5576     die "ERROR: archive contaions no data\n" if !$firstfile;
5577     chomp $firstfile;
5578
5579     return $firstfile;
5580 }
5581
5582 sub tar_restore_cleanup {
5583     my ($storecfg, $statfile) = @_;
5584
5585     print STDERR "starting cleanup\n";
5586
5587     if (my $fd = IO::File->new($statfile, "r")) {
5588         while (defined(my $line = <$fd>)) {
5589             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5590                 my $volid = $2;
5591                 eval {
5592                     if ($volid =~ m|^/|) {
5593                         unlink $volid || die 'unlink failed\n';
5594                     } else {
5595                         PVE::Storage::vdisk_free($storecfg, $volid);
5596                     }
5597                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5598                 };
5599                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5600             } else {
5601                 print STDERR "unable to parse line in statfile - $line";
5602             }
5603         }
5604         $fd->close();
5605     }
5606 }
5607
5608 sub restore_archive {
5609     my ($archive, $vmid, $user, $opts) = @_;
5610
5611     my $format = $opts->{format};
5612     my $comp;
5613
5614     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5615         $format = 'tar' if !$format;
5616         $comp = 'gzip';
5617     } elsif ($archive =~ m/\.tar$/) {
5618         $format = 'tar' if !$format;
5619     } elsif ($archive =~ m/.tar.lzo$/) {
5620         $format = 'tar' if !$format;
5621         $comp = 'lzop';
5622     } elsif ($archive =~ m/\.vma$/) {
5623         $format = 'vma' if !$format;
5624     } elsif ($archive =~ m/\.vma\.gz$/) {
5625         $format = 'vma' if !$format;
5626         $comp = 'gzip';
5627     } elsif ($archive =~ m/\.vma\.lzo$/) {
5628         $format = 'vma' if !$format;
5629         $comp = 'lzop';
5630     } else {
5631         $format = 'vma' if !$format; # default
5632     }
5633
5634     # try to detect archive format
5635     if ($format eq 'tar') {
5636         return restore_tar_archive($archive, $vmid, $user, $opts);
5637     } else {
5638         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5639     }
5640 }
5641
5642 sub restore_update_config_line {
5643     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5644
5645     return if $line =~ m/^\#qmdump\#/;
5646     return if $line =~ m/^\#vzdump\#/;
5647     return if $line =~ m/^lock:/;
5648     return if $line =~ m/^unused\d+:/;
5649     return if $line =~ m/^parent:/;
5650     return if $line =~ m/^template:/; # restored VM is never a template
5651
5652     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5653     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5654         # try to convert old 1.X settings
5655         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5656         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5657             my ($model, $macaddr) = split(/\=/, $devconfig);
5658             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5659             my $net = {
5660                 model => $model,
5661                 bridge => "vmbr$ind",
5662                 macaddr => $macaddr,
5663             };
5664             my $netstr = print_net($net);
5665
5666             print $outfd "net$cookie->{netcount}: $netstr\n";
5667             $cookie->{netcount}++;
5668         }
5669     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5670         my ($id, $netstr) = ($1, $2);
5671         my $net = parse_net($netstr);
5672         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5673         $netstr = print_net($net);
5674         print $outfd "$id: $netstr\n";
5675     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5676         my $virtdev = $1;
5677         my $value = $3;
5678         my $di = parse_drive($virtdev, $value);
5679         if (defined($di->{backup}) && !$di->{backup}) {
5680             print $outfd "#$line";
5681         } elsif ($map->{$virtdev}) {
5682             delete $di->{format}; # format can change on restore
5683             $di->{file} = $map->{$virtdev};
5684             $value = print_drive($vmid, $di);
5685             print $outfd "$virtdev: $value\n";
5686         } else {
5687             print $outfd $line;
5688         }
5689     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5690         my $vmgenid = $1;
5691         if ($vmgenid ne '0') {
5692             # always generate a new vmgenid if there was a valid one setup
5693             $vmgenid = generate_uuid();
5694         }
5695         print $outfd "vmgenid: $vmgenid\n";
5696     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5697         my ($uuid, $uuid_str);
5698         UUID::generate($uuid);
5699         UUID::unparse($uuid, $uuid_str);
5700         my $smbios1 = parse_smbios1($2);
5701         $smbios1->{uuid} = $uuid_str;
5702         print $outfd $1.print_smbios1($smbios1)."\n";
5703     } else {
5704         print $outfd $line;
5705     }
5706 }
5707
5708 sub scan_volids {
5709     my ($cfg, $vmid) = @_;
5710
5711     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5712
5713     my $volid_hash = {};
5714     foreach my $storeid (keys %$info) {
5715         foreach my $item (@{$info->{$storeid}}) {
5716             next if !($item->{volid} && $item->{size});
5717             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5718             $volid_hash->{$item->{volid}} = $item;
5719         }
5720     }
5721
5722     return $volid_hash;
5723 }
5724
5725 sub is_volume_in_use {
5726     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5727
5728     my $path = PVE::Storage::path($storecfg, $volid);
5729
5730     my $scan_config = sub {
5731         my ($cref, $snapname) = @_;
5732
5733         foreach my $key (keys %$cref) {
5734             my $value = $cref->{$key};
5735             if (is_valid_drivename($key)) {
5736                 next if $skip_drive && $key eq $skip_drive;
5737                 my $drive = parse_drive($key, $value);
5738                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5739                 return 1 if $volid eq $drive->{file};
5740                 if ($drive->{file} =~ m!^/!) {
5741                     return 1 if $drive->{file} eq $path;
5742                 } else {
5743                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5744                     next if !$storeid;
5745                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5746                     next if !$scfg;
5747                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5748                 }
5749             }
5750         }
5751
5752         return 0;
5753     };
5754
5755     return 1 if &$scan_config($conf);
5756
5757     undef $skip_drive;
5758
5759     foreach my $snapname (keys %{$conf->{snapshots}}) {
5760         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5761     }
5762
5763     return 0;
5764 }
5765
5766 sub update_disksize {
5767     my ($vmid, $conf, $volid_hash) = @_;
5768
5769     my $changes;
5770     my $prefix = "VM $vmid:";
5771
5772     # used and unused disks
5773     my $referenced = {};
5774
5775     # Note: it is allowed to define multiple storages with same path (alias), so
5776     # we need to check both 'volid' and real 'path' (two different volid can point
5777     # to the same path).
5778
5779     my $referencedpath = {};
5780
5781     # update size info
5782     foreach my $opt (keys %$conf) {
5783         if (is_valid_drivename($opt)) {
5784             my $drive = parse_drive($opt, $conf->{$opt});
5785             my $volid = $drive->{file};
5786             next if !$volid;
5787
5788             $referenced->{$volid} = 1;
5789             if ($volid_hash->{$volid} &&
5790                 (my $path = $volid_hash->{$volid}->{path})) {
5791                 $referencedpath->{$path} = 1;
5792             }
5793
5794             next if drive_is_cdrom($drive);
5795             next if !$volid_hash->{$volid};
5796
5797             $drive->{size} = $volid_hash->{$volid}->{size};
5798             my $new = print_drive($vmid, $drive);
5799             if ($new ne $conf->{$opt}) {
5800                 $changes = 1;
5801                 $conf->{$opt} = $new;
5802                 print "$prefix update disk '$opt' information.\n";
5803             }
5804         }
5805     }
5806
5807     # remove 'unusedX' entry if volume is used
5808     foreach my $opt (keys %$conf) {
5809         next if $opt !~ m/^unused\d+$/;
5810         my $volid = $conf->{$opt};
5811         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5812         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5813             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5814             $changes = 1;
5815             delete $conf->{$opt};
5816         }
5817
5818         $referenced->{$volid} = 1;
5819         $referencedpath->{$path} = 1 if $path;
5820     }
5821
5822     foreach my $volid (sort keys %$volid_hash) {
5823         next if $volid =~ m/vm-$vmid-state-/;
5824         next if $referenced->{$volid};
5825         my $path = $volid_hash->{$volid}->{path};
5826         next if !$path; # just to be sure
5827         next if $referencedpath->{$path};
5828         $changes = 1;
5829         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5830         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5831         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5832     }
5833
5834     return $changes;
5835 }
5836
5837 sub rescan {
5838     my ($vmid, $nolock, $dryrun) = @_;
5839
5840     my $cfg = PVE::Storage::config();
5841
5842     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5843     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5844     foreach my $stor (keys %{$cfg->{ids}}) {
5845         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5846     }
5847
5848     print "rescan volumes...\n";
5849     my $volid_hash = scan_volids($cfg, $vmid);
5850
5851     my $updatefn =  sub {
5852         my ($vmid) = @_;
5853
5854         my $conf = PVE::QemuConfig->load_config($vmid);
5855
5856         PVE::QemuConfig->check_lock($conf);
5857
5858         my $vm_volids = {};
5859         foreach my $volid (keys %$volid_hash) {
5860             my $info = $volid_hash->{$volid};
5861             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5862         }
5863
5864         my $changes = update_disksize($vmid, $conf, $vm_volids);
5865
5866         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5867     };
5868
5869     if (defined($vmid)) {
5870         if ($nolock) {
5871             &$updatefn($vmid);
5872         } else {
5873             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5874         }
5875     } else {
5876         my $vmlist = config_list();
5877         foreach my $vmid (keys %$vmlist) {
5878             if ($nolock) {
5879                 &$updatefn($vmid);
5880             } else {
5881                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5882             }
5883         }
5884     }
5885 }
5886
5887 sub restore_vma_archive {
5888     my ($archive, $vmid, $user, $opts, $comp) = @_;
5889
5890     my $readfrom = $archive;
5891
5892     my $cfg = PVE::Storage::config();
5893     my $commands = [];
5894     my $bwlimit = $opts->{bwlimit};
5895
5896     my $dbg_cmdstring = '';
5897     my $add_pipe = sub {
5898         my ($cmd) = @_;
5899         push @$commands, $cmd;
5900         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5901         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5902         $readfrom = '-';
5903     };
5904
5905     my $input = undef;
5906     if ($archive eq '-') {
5907         $input = '<&STDIN';
5908     } else {
5909         # If we use a backup from a PVE defined storage we also consider that
5910         # storage's rate limit:
5911         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5912         if (defined($volid)) {
5913             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5914             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5915             if ($readlimit) {
5916                 print STDERR "applying read rate limit: $readlimit\n";
5917                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5918                 $add_pipe->($cstream);
5919             }
5920         }
5921     }
5922
5923     if ($comp) {
5924         my $cmd;
5925         if ($comp eq 'gzip') {
5926             $cmd = ['zcat', $readfrom];
5927         } elsif ($comp eq 'lzop') {
5928             $cmd = ['lzop', '-d', '-c', $readfrom];
5929         } else {
5930             die "unknown compression method '$comp'\n";
5931         }
5932         $add_pipe->($cmd);
5933     }
5934
5935     my $tmpdir = "/var/tmp/vzdumptmp$$";
5936     rmtree $tmpdir;
5937
5938     # disable interrupts (always do cleanups)
5939     local $SIG{INT} =
5940         local $SIG{TERM} =
5941         local $SIG{QUIT} =
5942         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5943
5944     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5945     POSIX::mkfifo($mapfifo, 0600);
5946     my $fifofh;
5947
5948     my $openfifo = sub {
5949         open($fifofh, '>', $mapfifo) || die $!;
5950     };
5951
5952     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5953
5954     my $oldtimeout;
5955     my $timeout = 5;
5956
5957     my $devinfo = {};
5958
5959     my $rpcenv = PVE::RPCEnvironment::get();
5960
5961     my $conffile = PVE::QemuConfig->config_file($vmid);
5962     my $tmpfn = "$conffile.$$.tmp";
5963
5964     # Note: $oldconf is undef if VM does not exists
5965     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5966     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5967
5968     my %storage_limits;
5969
5970     my $print_devmap = sub {
5971         my $virtdev_hash = {};
5972
5973         my $cfgfn = "$tmpdir/qemu-server.conf";
5974
5975         # we can read the config - that is already extracted
5976         my $fh = IO::File->new($cfgfn, "r") ||
5977             "unable to read qemu-server.conf - $!\n";
5978
5979         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5980         if (-f $fwcfgfn) {
5981             my $pve_firewall_dir = '/etc/pve/firewall';
5982             mkdir $pve_firewall_dir; # make sure the dir exists
5983             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5984         }
5985
5986         while (defined(my $line = <$fh>)) {
5987             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5988                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5989                 die "archive does not contain data for drive '$virtdev'\n"
5990                     if !$devinfo->{$devname};
5991                 if (defined($opts->{storage})) {
5992                     $storeid = $opts->{storage} || 'local';
5993                 } elsif (!$storeid) {
5994                     $storeid = 'local';
5995                 }
5996                 $format = 'raw' if !$format;
5997                 $devinfo->{$devname}->{devname} = $devname;
5998                 $devinfo->{$devname}->{virtdev} = $virtdev;
5999                 $devinfo->{$devname}->{format} = $format;
6000                 $devinfo->{$devname}->{storeid} = $storeid;
6001
6002                 # check permission on storage
6003                 my $pool = $opts->{pool}; # todo: do we need that?
6004                 if ($user ne 'root@pam') {
6005                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6006                 }
6007
6008                 $storage_limits{$storeid} = $bwlimit;
6009
6010                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6011             }
6012         }
6013
6014         foreach my $key (keys %storage_limits) {
6015             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6016             next if !$limit;
6017             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6018             $storage_limits{$key} = $limit * 1024;
6019         }
6020
6021         foreach my $devname (keys %$devinfo) {
6022             die "found no device mapping information for device '$devname'\n"
6023                 if !$devinfo->{$devname}->{virtdev};
6024         }
6025
6026         # create empty/temp config
6027         if ($oldconf) {
6028             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6029             foreach_drive($oldconf, sub {
6030                 my ($ds, $drive) = @_;
6031
6032                 return if drive_is_cdrom($drive);
6033
6034                 my $volid = $drive->{file};
6035
6036                 return if !$volid || $volid =~ m|^/|;
6037
6038                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6039                 return if !$path || !$owner || ($owner != $vmid);
6040
6041                 # Note: only delete disk we want to restore
6042                 # other volumes will become unused
6043                 if ($virtdev_hash->{$ds}) {
6044                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6045                     if (my $err = $@) {
6046                         warn $err;
6047                     }
6048                 }
6049             });
6050
6051             # delete vmstate files
6052             # since after the restore we have no snapshots anymore
6053             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6054                 my $snap = $oldconf->{snapshots}->{$snapname};
6055                 if ($snap->{vmstate}) {
6056                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6057                     if (my $err = $@) {
6058                         warn $err;
6059                     }
6060                 }
6061             }
6062         }
6063
6064         my $map = {};
6065         foreach my $virtdev (sort keys %$virtdev_hash) {
6066             my $d = $virtdev_hash->{$virtdev};
6067             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6068             my $storeid = $d->{storeid};
6069             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6070
6071             my $map_opts = '';
6072             if (my $limit = $storage_limits{$storeid}) {
6073                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6074             }
6075
6076             # test if requested format is supported
6077             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6078             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6079             $d->{format} = $defFormat if !$supported;
6080
6081             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6082                                                   $d->{format}, undef, $alloc_size);
6083             print STDERR "new volume ID is '$volid'\n";
6084             $d->{volid} = $volid;
6085             my $path = PVE::Storage::path($cfg, $volid);
6086
6087             PVE::Storage::activate_volumes($cfg,[$volid]);
6088
6089             my $write_zeros = 1;
6090             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6091                 $write_zeros = 0;
6092             }
6093
6094             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6095
6096             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6097             $map->{$virtdev} = $volid;
6098         }
6099
6100         $fh->seek(0, 0) || die "seek failed - $!\n";
6101
6102         my $outfd = new IO::File ($tmpfn, "w") ||
6103             die "unable to write config for VM $vmid\n";
6104
6105         my $cookie = { netcount => 0 };
6106         while (defined(my $line = <$fh>)) {
6107             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6108         }
6109
6110         $fh->close();
6111         $outfd->close();
6112     };
6113
6114     eval {
6115         # enable interrupts
6116         local $SIG{INT} =
6117             local $SIG{TERM} =
6118             local $SIG{QUIT} =
6119             local $SIG{HUP} =
6120             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6121         local $SIG{ALRM} = sub { die "got timeout\n"; };
6122
6123         $oldtimeout = alarm($timeout);
6124
6125         my $parser = sub {
6126             my $line = shift;
6127
6128             print "$line\n";
6129
6130             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6131                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6132                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6133             } elsif ($line =~ m/^CTIME: /) {
6134                 # we correctly received the vma config, so we can disable
6135                 # the timeout now for disk allocation (set to 10 minutes, so
6136                 # that we always timeout if something goes wrong)
6137                 alarm(600);
6138                 &$print_devmap();
6139                 print $fifofh "done\n";
6140                 my $tmp = $oldtimeout || 0;
6141                 $oldtimeout = undef;
6142                 alarm($tmp);
6143                 close($fifofh);
6144             }
6145         };
6146
6147         print "restore vma archive: $dbg_cmdstring\n";
6148         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6149     };
6150     my $err = $@;
6151
6152     alarm($oldtimeout) if $oldtimeout;
6153
6154     my $vollist = [];
6155     foreach my $devname (keys %$devinfo) {
6156         my $volid = $devinfo->{$devname}->{volid};
6157         push @$vollist, $volid if $volid;
6158     }
6159
6160     PVE::Storage::deactivate_volumes($cfg, $vollist);
6161
6162     unlink $mapfifo;
6163
6164     if ($err) {
6165         rmtree $tmpdir;
6166         unlink $tmpfn;
6167
6168         foreach my $devname (keys %$devinfo) {
6169             my $volid = $devinfo->{$devname}->{volid};
6170             next if !$volid;
6171             eval {
6172                 if ($volid =~ m|^/|) {
6173                     unlink $volid || die 'unlink failed\n';
6174                 } else {
6175                     PVE::Storage::vdisk_free($cfg, $volid);
6176                 }
6177                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6178             };
6179             print STDERR "unable to cleanup '$volid' - $@" if $@;
6180         }
6181         die $err;
6182     }
6183
6184     rmtree $tmpdir;
6185
6186     rename($tmpfn, $conffile) ||
6187         die "unable to commit configuration file '$conffile'\n";
6188
6189     PVE::Cluster::cfs_update(); # make sure we read new file
6190
6191     eval { rescan($vmid, 1); };
6192     warn $@ if $@;
6193 }
6194
6195 sub restore_tar_archive {
6196     my ($archive, $vmid, $user, $opts) = @_;
6197
6198     if ($archive ne '-') {
6199         my $firstfile = tar_archive_read_firstfile($archive);
6200         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6201             if $firstfile ne 'qemu-server.conf';
6202     }
6203
6204     my $storecfg = PVE::Storage::config();
6205
6206     # destroy existing data - keep empty config
6207     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6208     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6209
6210     my $tocmd = "/usr/lib/qemu-server/qmextract";
6211
6212     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6213     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6214     $tocmd .= ' --prealloc' if $opts->{prealloc};
6215     $tocmd .= ' --info' if $opts->{info};
6216
6217     # tar option "xf" does not autodetect compression when read from STDIN,
6218     # so we pipe to zcat
6219     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6220         PVE::Tools::shellquote("--to-command=$tocmd");
6221
6222     my $tmpdir = "/var/tmp/vzdumptmp$$";
6223     mkpath $tmpdir;
6224
6225     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6226     local $ENV{VZDUMP_VMID} = $vmid;
6227     local $ENV{VZDUMP_USER} = $user;
6228
6229     my $conffile = PVE::QemuConfig->config_file($vmid);
6230     my $tmpfn = "$conffile.$$.tmp";
6231
6232     # disable interrupts (always do cleanups)
6233     local $SIG{INT} =
6234         local $SIG{TERM} =
6235         local $SIG{QUIT} =
6236         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6237
6238     eval {
6239         # enable interrupts
6240         local $SIG{INT} =
6241             local $SIG{TERM} =
6242             local $SIG{QUIT} =
6243             local $SIG{HUP} =
6244             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6245
6246         if ($archive eq '-') {
6247             print "extracting archive from STDIN\n";
6248             run_command($cmd, input => "<&STDIN");
6249         } else {
6250             print "extracting archive '$archive'\n";
6251             run_command($cmd);
6252         }
6253
6254         return if $opts->{info};
6255
6256         # read new mapping
6257         my $map = {};
6258         my $statfile = "$tmpdir/qmrestore.stat";
6259         if (my $fd = IO::File->new($statfile, "r")) {
6260             while (defined (my $line = <$fd>)) {
6261                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6262                     $map->{$1} = $2 if $1;
6263                 } else {
6264                     print STDERR "unable to parse line in statfile - $line\n";
6265                 }
6266             }
6267             $fd->close();
6268         }
6269
6270         my $confsrc = "$tmpdir/qemu-server.conf";
6271
6272         my $srcfd = new IO::File($confsrc, "r") ||
6273             die "unable to open file '$confsrc'\n";
6274
6275         my $outfd = new IO::File ($tmpfn, "w") ||
6276             die "unable to write config for VM $vmid\n";
6277
6278         my $cookie = { netcount => 0 };
6279         while (defined (my $line = <$srcfd>)) {
6280             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6281         }
6282
6283         $srcfd->close();
6284         $outfd->close();
6285     };
6286     my $err = $@;
6287
6288     if ($err) {
6289
6290         unlink $tmpfn;
6291
6292         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6293
6294         die $err;
6295     }
6296
6297     rmtree $tmpdir;
6298
6299     rename $tmpfn, $conffile ||
6300         die "unable to commit configuration file '$conffile'\n";
6301
6302     PVE::Cluster::cfs_update(); # make sure we read new file
6303
6304     eval { rescan($vmid, 1); };
6305     warn $@ if $@;
6306 };
6307
6308 sub foreach_storage_used_by_vm {
6309     my ($conf, $func) = @_;
6310
6311     my $sidhash = {};
6312
6313     foreach_drive($conf, sub {
6314         my ($ds, $drive) = @_;
6315         return if drive_is_cdrom($drive);
6316
6317         my $volid = $drive->{file};
6318
6319         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6320         $sidhash->{$sid} = $sid if $sid;
6321     });
6322
6323     foreach my $sid (sort keys %$sidhash) {
6324         &$func($sid);
6325     }
6326 }
6327
6328 sub do_snapshots_with_qemu {
6329     my ($storecfg, $volid) = @_;
6330
6331     my $storage_name = PVE::Storage::parse_volume_id($volid);
6332
6333     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6334         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6335         return 1;
6336     }
6337
6338     if ($volid =~ m/\.(qcow2|qed)$/){
6339         return 1;
6340     }
6341
6342     return undef;
6343 }
6344
6345 sub qga_check_running {
6346     my ($vmid, $nowarn) = @_;
6347
6348     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6349     if ($@) {
6350         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6351         return 0;
6352     }
6353     return 1;
6354 }
6355
6356 sub template_create {
6357     my ($vmid, $conf, $disk) = @_;
6358
6359     my $storecfg = PVE::Storage::config();
6360
6361     foreach_drive($conf, sub {
6362         my ($ds, $drive) = @_;
6363
6364         return if drive_is_cdrom($drive);
6365         return if $disk && $ds ne $disk;
6366
6367         my $volid = $drive->{file};
6368         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6369
6370         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6371         $drive->{file} = $voliddst;
6372         $conf->{$ds} = print_drive($vmid, $drive);
6373         PVE::QemuConfig->write_config($vmid, $conf);
6374     });
6375 }
6376
6377 sub qemu_img_convert {
6378     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6379
6380     my $storecfg = PVE::Storage::config();
6381     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6382     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6383
6384     if ($src_storeid && $dst_storeid) {
6385
6386         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6387
6388         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6389         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6390
6391         my $src_format = qemu_img_format($src_scfg, $src_volname);
6392         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6393
6394         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6395         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6396
6397         my $cmd = [];
6398         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6399         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6400         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6401         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6402         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6403         if ($is_zero_initialized) {
6404             push @$cmd, "zeroinit:$dst_path";
6405         } else {
6406             push @$cmd, $dst_path;
6407         }
6408
6409         my $parser = sub {
6410             my $line = shift;
6411             if($line =~ m/\((\S+)\/100\%\)/){
6412                 my $percent = $1;
6413                 my $transferred = int($size * $percent / 100);
6414                 my $remaining = $size - $transferred;
6415
6416                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6417             }
6418
6419         };
6420
6421         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6422         my $err = $@;
6423         die "copy failed: $err" if $err;
6424     }
6425 }
6426
6427 sub qemu_img_format {
6428     my ($scfg, $volname) = @_;
6429
6430     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6431         return $1;
6432     } else {
6433         return "raw";
6434     }
6435 }
6436
6437 sub qemu_drive_mirror {
6438     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6439
6440     $jobs = {} if !$jobs;
6441
6442     my $qemu_target;
6443     my $format;
6444     $jobs->{"drive-$drive"} = {};
6445
6446     if ($dst_volid =~ /^nbd:/) {
6447         $qemu_target = $dst_volid;
6448         $format = "nbd";
6449     } else {
6450         my $storecfg = PVE::Storage::config();
6451         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6452
6453         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6454
6455         $format = qemu_img_format($dst_scfg, $dst_volname);
6456
6457         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6458
6459         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6460     }
6461
6462     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6463     $opts->{format} = $format if $format;
6464
6465     print "drive mirror is starting for drive-$drive\n";
6466
6467     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6468
6469     if (my $err = $@) {
6470         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6471         die "mirroring error: $err";
6472     }
6473
6474     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6475 }
6476
6477 sub qemu_drive_mirror_monitor {
6478     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6479
6480     eval {
6481         my $err_complete = 0;
6482
6483         while (1) {
6484             die "storage migration timed out\n" if $err_complete > 300;
6485
6486             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6487
6488             my $running_mirror_jobs = {};
6489             foreach my $stat (@$stats) {
6490                 next if $stat->{type} ne 'mirror';
6491                 $running_mirror_jobs->{$stat->{device}} = $stat;
6492             }
6493
6494             my $readycounter = 0;
6495
6496             foreach my $job (keys %$jobs) {
6497
6498                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6499                     print "$job : finished\n";
6500                     delete $jobs->{$job};
6501                     next;
6502                 }
6503
6504                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6505
6506                 my $busy = $running_mirror_jobs->{$job}->{busy};
6507                 my $ready = $running_mirror_jobs->{$job}->{ready};
6508                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6509                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6510                     my $remaining = $total - $transferred;
6511                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6512
6513                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6514                 }
6515
6516                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6517             }
6518
6519             last if scalar(keys %$jobs) == 0;
6520
6521             if ($readycounter == scalar(keys %$jobs)) {
6522                 print "all mirroring jobs are ready \n";
6523                 last if $skipcomplete; #do the complete later
6524
6525                 if ($vmiddst && $vmiddst != $vmid) {
6526                     my $agent_running = $qga && qga_check_running($vmid);
6527                     if ($agent_running) {
6528                         print "freeze filesystem\n";
6529                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6530                     } else {
6531                         print "suspend vm\n";
6532                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6533                     }
6534
6535                     # if we clone a disk for a new target vm, we don't switch the disk
6536                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6537
6538                     if ($agent_running) {
6539                         print "unfreeze filesystem\n";
6540                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6541                     } else {
6542                         print "resume vm\n";
6543                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6544                     }
6545
6546                     last;
6547                 } else {
6548
6549                     foreach my $job (keys %$jobs) {
6550                         # try to switch the disk if source and destination are on the same guest
6551                         print "$job: Completing block job...\n";
6552
6553                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6554                         if ($@ =~ m/cannot be completed/) {
6555                             print "$job: Block job cannot be completed, try again.\n";
6556                             $err_complete++;
6557                         }else {
6558                             print "$job: Completed successfully.\n";
6559                             $jobs->{$job}->{complete} = 1;
6560                         }
6561                     }
6562                 }
6563             }
6564             sleep 1;
6565         }
6566     };
6567     my $err = $@;
6568
6569     if ($err) {
6570         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6571         die "mirroring error: $err";
6572     }
6573
6574 }
6575
6576 sub qemu_blockjobs_cancel {
6577     my ($vmid, $jobs) = @_;
6578
6579     foreach my $job (keys %$jobs) {
6580         print "$job: Cancelling block job\n";
6581         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6582         $jobs->{$job}->{cancel} = 1;
6583     }
6584
6585     while (1) {
6586         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6587
6588         my $running_jobs = {};
6589         foreach my $stat (@$stats) {
6590             $running_jobs->{$stat->{device}} = $stat;
6591         }
6592
6593         foreach my $job (keys %$jobs) {
6594
6595             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6596                 print "$job: Done.\n";
6597                 delete $jobs->{$job};
6598             }
6599         }
6600
6601         last if scalar(keys %$jobs) == 0;
6602
6603         sleep 1;
6604     }
6605 }
6606
6607 sub clone_disk {
6608     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6609         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6610
6611     my $newvolid;
6612
6613     if (!$full) {
6614         print "create linked clone of drive $drivename ($drive->{file})\n";
6615         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6616         push @$newvollist, $newvolid;
6617     } else {
6618
6619         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6620         $storeid = $storage if $storage;
6621
6622         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6623         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6624
6625         print "create full clone of drive $drivename ($drive->{file})\n";
6626         my $name = undef;
6627         if (drive_is_cloudinit($drive)) {
6628             $name = "vm-$newvmid-cloudinit";
6629             # cloudinit only supports raw and qcow2 atm:
6630             if ($dst_format eq 'qcow2') {
6631                 $name .= '.qcow2';
6632             } elsif ($dst_format ne 'raw') {
6633                 die "clone: unhandled format for cloudinit image\n";
6634             }
6635         }
6636         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6637         push @$newvollist, $newvolid;
6638
6639         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6640
6641         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6642         if (!$running || $snapname) {
6643             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6644         } else {
6645
6646             my $kvmver = get_running_qemu_version ($vmid);
6647             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6648                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6649                     if $drive->{iothread};
6650             }
6651
6652             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6653         }
6654     }
6655
6656     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6657
6658     my $disk = $drive;
6659     $disk->{format} = undef;
6660     $disk->{file} = $newvolid;
6661     $disk->{size} = $size;
6662
6663     return $disk;
6664 }
6665
6666 # this only works if VM is running
6667 sub get_current_qemu_machine {
6668     my ($vmid) = @_;
6669
6670     my $cmd = { execute => 'query-machines', arguments => {} };
6671     my $res = vm_qmp_command($vmid, $cmd);
6672
6673     my ($current, $default);
6674     foreach my $e (@$res) {
6675         $default = $e->{name} if $e->{'is-default'};
6676         $current = $e->{name} if $e->{'is-current'};
6677     }
6678
6679     # fallback to the default machine if current is not supported by qemu
6680     return $current || $default || 'pc';
6681 }
6682
6683 sub get_running_qemu_version {
6684     my ($vmid) = @_;
6685     my $cmd = { execute => 'query-version', arguments => {} };
6686     my $res = vm_qmp_command($vmid, $cmd);
6687     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6688 }
6689
6690 sub qemu_machine_feature_enabled {
6691     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6692
6693     my $current_major;
6694     my $current_minor;
6695
6696     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6697
6698         $current_major = $3;
6699         $current_minor = $4;
6700
6701     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6702
6703         $current_major = $1;
6704         $current_minor = $2;
6705     }
6706
6707     return 1 if $current_major > $version_major ||
6708                 ($current_major == $version_major &&
6709                  $current_minor >= $version_minor);
6710 }
6711
6712 sub qemu_machine_pxe {
6713     my ($vmid, $conf, $machine) = @_;
6714
6715     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6716
6717     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6718         $machine .= '.pxe';
6719     }
6720
6721     return $machine;
6722 }
6723
6724 sub qemu_use_old_bios_files {
6725     my ($machine_type) = @_;
6726
6727     return if !$machine_type;
6728
6729     my $use_old_bios_files = undef;
6730
6731     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6732         $machine_type = $1;
6733         $use_old_bios_files = 1;
6734     } else {
6735         my $kvmver = kvm_user_version();
6736         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6737         # load new efi bios files on migration. So this hack is required to allow
6738         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6739         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6740         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6741     }
6742
6743     return ($use_old_bios_files, $machine_type);
6744 }
6745
6746 sub create_efidisk {
6747     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6748
6749     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6750
6751     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6752     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6753     PVE::Storage::activate_volumes($storecfg, [$volid]);
6754
6755     my $path = PVE::Storage::path($storecfg, $volid);
6756     eval {
6757         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6758     };
6759     die "Copying EFI vars image failed: $@" if $@;
6760
6761     return ($volid, $vars_size);
6762 }
6763
6764 sub lspci {
6765
6766     my $devices = {};
6767
6768     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6769             my (undef, $id, $function) = @_;
6770             my $res = { id => $id, function => $function};
6771             push @{$devices->{$id}}, $res;
6772     });
6773
6774     # Entries should be sorted by functions.
6775     foreach my $id (keys %$devices) {
6776         my $dev = $devices->{$id};
6777         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6778     }
6779
6780     return $devices;
6781 }
6782
6783 sub vm_iothreads_list {
6784     my ($vmid) = @_;
6785
6786     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6787
6788     my $iothreads = {};
6789     foreach my $iothread (@$res) {
6790         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6791     }
6792
6793     return $iothreads;
6794 }
6795
6796 sub scsihw_infos {
6797     my ($conf, $drive) = @_;
6798
6799     my $maxdev = 0;
6800
6801     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6802         $maxdev = 7;
6803     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6804         $maxdev = 1;
6805     } else {
6806         $maxdev = 256;
6807     }
6808
6809     my $controller = int($drive->{index} / $maxdev);
6810     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6811
6812     return ($maxdev, $controller, $controller_prefix);
6813 }
6814
6815 sub add_hyperv_enlightenments {
6816     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6817
6818     return if $winversion < 6;
6819     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6820
6821     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6822
6823     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6824         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6825         push @$cpuFlags , 'hv_vapic';
6826         push @$cpuFlags , 'hv_time';
6827     } else {
6828         push @$cpuFlags , 'hv_spinlocks=0xffff';
6829     }
6830
6831     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6832         push @$cpuFlags , 'hv_reset';
6833         push @$cpuFlags , 'hv_vpindex';
6834         push @$cpuFlags , 'hv_runtime';
6835     }
6836
6837     if ($winversion >= 7) {
6838         push @$cpuFlags , 'hv_relaxed';
6839
6840         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6841             push @$cpuFlags , 'hv_synic';
6842             push @$cpuFlags , 'hv_stimer';
6843         }
6844     }
6845 }
6846
6847 sub windows_version {
6848     my ($ostype) = @_;
6849
6850     return 0 if !$ostype;
6851
6852     my $winversion = 0;
6853
6854     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6855         $winversion = 5;
6856     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6857         $winversion = 6;
6858     } elsif ($ostype =~ m/^win(\d+)$/) {
6859         $winversion = $1;
6860     }
6861
6862     return $winversion;
6863 }
6864
6865 sub resolve_dst_disk_format {
6866         my ($storecfg, $storeid, $src_volname, $format) = @_;
6867         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6868
6869         if (!$format) {
6870             # if no target format is specified, use the source disk format as hint
6871             if ($src_volname) {
6872                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6873                 $format = qemu_img_format($scfg, $src_volname);
6874             } else {
6875                 return $defFormat;
6876             }
6877         }
6878
6879         # test if requested format is supported - else use default
6880         my $supported = grep { $_ eq $format } @$validFormats;
6881         $format = $defFormat if !$supported;
6882         return $format;
6883 }
6884
6885 sub resolve_first_disk {
6886     my $conf = shift;
6887     my @disks = PVE::QemuServer::valid_drive_names();
6888     my $firstdisk;
6889     foreach my $ds (reverse @disks) {
6890         next if !$conf->{$ds};
6891         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6892         next if PVE::QemuServer::drive_is_cdrom($disk);
6893         $firstdisk = $ds;
6894     }
6895     return $firstdisk;
6896 }
6897
6898 sub generate_uuid {
6899     my ($uuid, $uuid_str);
6900     UUID::generate($uuid);
6901     UUID::unparse($uuid, $uuid_str);
6902     return $uuid_str;
6903 }
6904
6905 sub generate_smbios1_uuid {
6906     return "uuid=".generate_uuid();
6907 }
6908
6909 # bash completion helper
6910
6911 sub complete_backup_archives {
6912     my ($cmdname, $pname, $cvalue) = @_;
6913
6914     my $cfg = PVE::Storage::config();
6915
6916     my $storeid;
6917
6918     if ($cvalue =~ m/^([^:]+):/) {
6919         $storeid = $1;
6920     }
6921
6922     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6923
6924     my $res = [];
6925     foreach my $id (keys %$data) {
6926         foreach my $item (@{$data->{$id}}) {
6927             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6928             push @$res, $item->{volid} if defined($item->{volid});
6929         }
6930     }
6931
6932     return $res;
6933 }
6934
6935 my $complete_vmid_full = sub {
6936     my ($running) = @_;
6937
6938     my $idlist = vmstatus();
6939
6940     my $res = [];
6941
6942     foreach my $id (keys %$idlist) {
6943         my $d = $idlist->{$id};
6944         if (defined($running)) {
6945             next if $d->{template};
6946             next if $running && $d->{status} ne 'running';
6947             next if !$running && $d->{status} eq 'running';
6948         }
6949         push @$res, $id;
6950
6951     }
6952     return $res;
6953 };
6954
6955 sub complete_vmid {
6956     return &$complete_vmid_full();
6957 }
6958
6959 sub complete_vmid_stopped {
6960     return &$complete_vmid_full(0);
6961 }
6962
6963 sub complete_vmid_running {
6964     return &$complete_vmid_full(1);
6965 }
6966
6967 sub complete_storage {
6968
6969     my $cfg = PVE::Storage::config();
6970     my $ids = $cfg->{ids};
6971
6972     my $res = [];
6973     foreach my $sid (keys %$ids) {
6974         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6975         next if !$ids->{$sid}->{content}->{images};
6976         push @$res, $sid;
6977     }
6978
6979     return $res;
6980 }
6981
6982 sub nbd_stop {
6983     my ($vmid) = @_;
6984
6985     vm_mon_cmd($vmid, 'nbd-server-stop');
6986 }
6987
6988 1;