PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $vga_fmt = {
 227     type => {
 228         description => "Select the VGA type.",
 229         type => 'string',
 230         default => 'std',
 231         optional => 1,
 232         default_key => 1,
 233         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 234     },
 235     memory => {
 236         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 237         type => 'integer',
 238         optional => 1,
 239         minimum => 4,
 240         maximum => 512,
 241     },
 242 };
 243
 244 my $confdesc = {
 245     onboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Specifies whether a VM will be started during system bootup.",
 249         default => 0,
 250     },
 251     autostart => {
 252         optional => 1,
 253         type => 'boolean',
 254         description => "Automatic restart after crash (currently ignored).",
 255         default => 0,
 256     },
 257     hotplug => {
 258         optional => 1,
 259         type => 'string', format => 'pve-hotplug-features',
 260         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 261         default => 'network,disk,usb',
 262     },
 263     reboot => {
 264         optional => 1,
 265         type => 'boolean',
 266         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 267         default => 1,
 268     },
 269     lock => {
 270         optional => 1,
 271         type => 'string',
 272         description => "Lock/unlock the VM.",
 273         enum => [qw(migrate backup snapshot rollback)],
 274     },
 275     cpulimit => {
 276         optional => 1,
 277         type => 'number',
 278         description => "Limit of CPU usage.",
 279         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 280         minimum => 0,
 281         maximum => 128,
 282         default => 0,
 283     },
 284     cpuunits => {
 285         optional => 1,
 286         type => 'integer',
 287         description => "CPU weight for a VM.",
 288         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 289         minimum => 2,
 290         maximum => 262144,
 291         default => 1024,
 292     },
 293     memory => {
 294         optional => 1,
 295         type => 'integer',
 296         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 297         minimum => 16,
 298         default => 512,
 299     },
 300     balloon => {
 301         optional => 1,
 302         type => 'integer',
 303         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 304         minimum => 0,
 305     },
 306     shares => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 310         minimum => 0,
 311         maximum => 50000,
 312         default => 1000,
 313     },
 314     keyboard => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 318                        "It should not be necessary to set it.",
 319         enum => PVE::Tools::kvmkeymaplist(),
 320         default => undef,
 321     },
 322     name => {
 323         optional => 1,
 324         type => 'string', format => 'dns-name',
 325         description => "Set a name for the VM. Only used on the configuration web interface.",
 326     },
 327     scsihw => {
 328         optional => 1,
 329         type => 'string',
 330         description => "SCSI controller model",
 331         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 332         default => 'lsi',
 333     },
 334     description => {
 335         optional => 1,
 336         type => 'string',
 337         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 338     },
 339     ostype => {
 340         optional => 1,
 341         type => 'string',
 342         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 343         description => "Specify guest operating system.",
 344         verbose_description => <<EODESC,
 345 Specify guest operating system. This is used to enable special
 346 optimization/features for specific operating systems:
 347
 348 [horizontal]
 349 other;; unspecified OS
 350 wxp;; Microsoft Windows XP
 351 w2k;; Microsoft Windows 2000
 352 w2k3;; Microsoft Windows 2003
 353 w2k8;; Microsoft Windows 2008
 354 wvista;; Microsoft Windows Vista
 355 win7;; Microsoft Windows 7
 356 win8;; Microsoft Windows 8/2012/2012r2
 357 win10;; Microsoft Windows 10/2016
 358 l24;; Linux 2.4 Kernel
 359 l26;; Linux 2.6/3.X Kernel
 360 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 361 EODESC
 362     },
 363     boot => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 367         pattern => '[acdn]{1,4}',
 368         default => 'cdn',
 369     },
 370     bootdisk => {
 371         optional => 1,
 372         type => 'string', format => 'pve-qm-bootdisk',
 373         description => "Enable booting from specified disk.",
 374         pattern => '(ide|sata|scsi|virtio)\d+',
 375     },
 376     smp => {
 377         optional => 1,
 378         type => 'integer',
 379         description => "The number of CPUs. Please use option -sockets instead.",
 380         minimum => 1,
 381         default => 1,
 382     },
 383     sockets => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "The number of CPU sockets.",
 387         minimum => 1,
 388         default => 1,
 389     },
 390     cores => {
 391         optional => 1,
 392         type => 'integer',
 393         description => "The number of cores per socket.",
 394         minimum => 1,
 395         default => 1,
 396     },
 397     numa => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Enable/disable NUMA.",
 401         default => 0,
 402     },
 403     hugepages => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Enable/disable hugepages memory.",
 407         enum => [qw(any 2 1024)],
 408     },
 409     vcpus => {
 410         optional => 1,
 411         type => 'integer',
 412         description => "Number of hotplugged vcpus.",
 413         minimum => 1,
 414         default => 0,
 415     },
 416     acpi => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable ACPI.",
 420         default => 1,
 421     },
 422     agent => {
 423         optional => 1,
 424         description => "Enable/disable Qemu GuestAgent and its properties.",
 425         type => 'string',
 426         format => $agent_fmt,
 427     },
 428     kvm => {
 429         optional => 1,
 430         type => 'boolean',
 431         description => "Enable/disable KVM hardware virtualization.",
 432         default => 1,
 433     },
 434     tdf => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable time drift fix.",
 438         default => 0,
 439     },
 440     localtime => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 444     },
 445     freeze => {
 446         optional => 1,
 447         type => 'boolean',
 448         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 449     },
 450     vga => {
 451         optional => 1,
 452         type => 'string', format => $vga_fmt,
 453         description => "Configure the VGA hardware.",
 454         verbose_description => "Configure the VGA Hardware. If you want to use ".
 455             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 456             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 457             "is 'std' for all OS types besides some Windows versions (XP and " .
 458             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 459             "display server. For win* OS you can select how many independent " .
 460             "displays you want, Linux guests can add displays them self.\n".
 461             "You can also run without any graphic card, using a serial device as terminal.",
 462     },
 463     watchdog => {
 464         optional => 1,
 465         type => 'string', format => 'pve-qm-watchdog',
 466         description => "Create a virtual hardware watchdog device.",
 467         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 468             " (by a guest action), the watchdog must be periodically polled " .
 469             "by an agent inside the guest or else the watchdog will reset " .
 470             "the guest (or execute the respective action specified)",
 471     },
 472     startdate => {
 473         optional => 1,
 474         type => 'string',
 475         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 476         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 477         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 478         default => 'now',
 479     },
 480     startup =>  get_standard_option('pve-startup-order'),
 481     template => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable Template.",
 485         default => 0,
 486     },
 487     args => {
 488         optional => 1,
 489         type => 'string',
 490         description => "Arbitrary arguments passed to kvm.",
 491         verbose_description => <<EODESCR,
 492 Arbitrary arguments passed to kvm, for example:
 493
 494 args: -no-reboot -no-hpet
 495
 496 NOTE: this option is for experts only.
 497 EODESCR
 498     },
 499     tablet => {
 500         optional => 1,
 501         type => 'boolean',
 502         default => 1,
 503         description => "Enable/disable the USB tablet device.",
 504         verbose_description => "Enable/disable the USB tablet device. This device is " .
 505             "usually needed to allow absolute mouse positioning with VNC. " .
 506             "Else the mouse runs out of sync with normal VNC clients. " .
 507             "If you're running lots of console-only guests on one host, " .
 508             "you may consider disabling this to save some context switches. " .
 509             "This is turned off by default if you use spice (-vga=qxl).",
 510     },
 511     migrate_speed => {
 512         optional => 1,
 513         type => 'integer',
 514         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 515         minimum => 0,
 516         default => 0,
 517     },
 518     migrate_downtime => {
 519         optional => 1,
 520         type => 'number',
 521         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 522         minimum => 0,
 523         default => 0.1,
 524     },
 525     cdrom => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-ide',
 528         typetext => '<volume>',
 529         description => "This is an alias for option -ide2",
 530     },
 531     cpu => {
 532         optional => 1,
 533         description => "Emulated CPU type.",
 534         type => 'string',
 535         format => $cpu_fmt,
 536     },
 537     parent => get_standard_option('pve-snapshot-name', {
 538         optional => 1,
 539         description => "Parent snapshot name. This is used internally, and should not be modified.",
 540     }),
 541     snaptime => {
 542         optional => 1,
 543         description => "Timestamp for snapshots.",
 544         type => 'integer',
 545         minimum => 0,
 546     },
 547     vmstate => {
 548         optional => 1,
 549         type => 'string', format => 'pve-volume-id',
 550         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 551     },
 552     vmstatestorage => get_standard_option('pve-storage-id', {
 553         description => "Default storage for VM state volumes/files.",
 554         optional => 1,
 555     }),
 556     runningmachine => get_standard_option('pve-qemu-machine', {
 557         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 558     }),
 559     machine => get_standard_option('pve-qemu-machine'),
 560     smbios1 => {
 561         description => "Specify SMBIOS type 1 fields.",
 562         type => 'string', format => 'pve-qm-smbios1',
 563         maxLength => 256,
 564         optional => 1,
 565     },
 566     protection => {
 567         optional => 1,
 568         type => 'boolean',
 569         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 570         default => 0,
 571     },
 572     bios => {
 573         optional => 1,
 574         type => 'string',
 575         enum => [ qw(seabios ovmf) ],
 576         description => "Select BIOS implementation.",
 577         default => 'seabios',
 578     },
 579     vmgenid => {
 580         type => 'string',
 581         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 582         format_description => 'UUID',
 583         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 584         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 585             " 128-bit integer value identifier to the guest OS. This allows to".
 586             " notify the guest operating system when the virtual machine is".
 587             " executed with a different configuration (e.g. snapshot execution".
 588             " or creation from a template). The guest operating system notices".
 589             " the change, and is then able to react as appropriate by marking".
 590             " its copies of distributed databases as dirty, re-initializing its".
 591             " random number generator, etc.\n".
 592             "Note that auto-creation only works when done throug API/CLI create".
 593             " or update methods, but not when manually editing the config file.",
 594         default => "1 (autogenerated)",
 595         optional => 1,
 596     },
 597 };
 598
 599 my $confdesc_cloudinit = {
 600     citype => {
 601         optional => 1,
 602         type => 'string',
 603         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 604         enum => ['configdrive2', 'nocloud'],
 605     },
 606     ciuser => {
 607         optional => 1,
 608         type => 'string',
 609         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 610     },
 611     cipassword => {
 612         optional => 1,
 613         type => 'string',
 614         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 615     },
 616     searchdomain => {
 617         optional => 1,
 618         type => 'string',
 619         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 620     },
 621     nameserver => {
 622         optional => 1,
 623         type => 'string', format => 'address-list',
 624         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 625     },
 626     sshkeys => {
 627         optional => 1,
 628         type => 'string',
 629         format => 'urlencoded',
 630         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 631     },
 632 };
 633
 634 # what about other qemu settings ?
 635 #cpu => 'string',
 636 #machine => 'string',
 637 #fda => 'file',
 638 #fdb => 'file',
 639 #mtdblock => 'file',
 640 #sd => 'file',
 641 #pflash => 'file',
 642 #snapshot => 'bool',
 643 #bootp => 'file',
 644 ##tftp => 'dir',
 645 ##smb => 'dir',
 646 #kernel => 'file',
 647 #append => 'string',
 648 #initrd => 'file',
 649 ##soundhw => 'string',
 650
 651 while (my ($k, $v) = each %$confdesc) {
 652     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 653 }
 654
 655 my $MAX_IDE_DISKS = 4;
 656 my $MAX_SCSI_DISKS = 14;
 657 my $MAX_VIRTIO_DISKS = 16;
 658 my $MAX_SATA_DISKS = 6;
 659 my $MAX_USB_DEVICES = 5;
 660 my $MAX_NETS = 32;
 661 my $MAX_UNUSED_DISKS = 8;
 662 my $MAX_HOSTPCI_DEVICES = 4;
 663 my $MAX_SERIAL_PORTS = 4;
 664 my $MAX_PARALLEL_PORTS = 3;
 665 my $MAX_NUMA = 8;
 666
 667 my $numa_fmt = {
 668     cpus => {
 669         type => "string",
 670         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 671         description => "CPUs accessing this NUMA node.",
 672         format_description => "id[-id];...",
 673     },
 674     memory => {
 675         type => "number",
 676         description => "Amount of memory this NUMA node provides.",
 677         optional => 1,
 678     },
 679     hostnodes => {
 680         type => "string",
 681         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 682         description => "Host NUMA nodes to use.",
 683         format_description => "id[-id];...",
 684         optional => 1,
 685     },
 686     policy => {
 687         type => 'string',
 688         enum => [qw(preferred bind interleave)],
 689         description => "NUMA allocation policy.",
 690         optional => 1,
 691     },
 692 };
 693 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 694 my $numadesc = {
 695     optional => 1,
 696     type => 'string', format => $numa_fmt,
 697     description => "NUMA topology.",
 698 };
 699 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 700
 701 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 702     $confdesc->{"numa$i"} = $numadesc;
 703 }
 704
 705 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 706                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 707                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 708 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 709
 710 my $net_fmt_bridge_descr = <<__EOD__;
 711 Bridge to attach the network device to. The Proxmox VE standard bridge
 712 is called 'vmbr0'.
 713
 714 If you do not specify a bridge, we create a kvm user (NATed) network
 715 device, which provides DHCP and DNS services. The following addresses
 716 are used:
 717
 718  10.0.2.2   Gateway
 719  10.0.2.3   DNS Server
 720  10.0.2.4   SMB Server
 721
 722 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 723 __EOD__
 724
 725 my $net_fmt = {
 726     macaddr => {
 727         type => 'string',
 728         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 729         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 730         format_description => "XX:XX:XX:XX:XX:XX",
 731         optional => 1,
 732     },
 733     model => {
 734         type => 'string',
 735         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 736         enum => $nic_model_list,
 737         default_key => 1,
 738     },
 739     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 740     bridge => {
 741         type => 'string',
 742         description => $net_fmt_bridge_descr,
 743         format_description => 'bridge',
 744         optional => 1,
 745     },
 746     queues => {
 747         type => 'integer',
 748         minimum => 0, maximum => 16,
 749         description => 'Number of packet queues to be used on the device.',
 750         optional => 1,
 751     },
 752     rate => {
 753         type => 'number',
 754         minimum => 0,
 755         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 756         optional => 1,
 757     },
 758     tag => {
 759         type => 'integer',
 760         minimum => 1, maximum => 4094,
 761         description => 'VLAN tag to apply to packets on this interface.',
 762         optional => 1,
 763     },
 764     trunks => {
 765         type => 'string',
 766         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 767         description => 'VLAN trunks to pass through this interface.',
 768         format_description => 'vlanid[;vlanid...]',
 769         optional => 1,
 770     },
 771     firewall => {
 772         type => 'boolean',
 773         description => 'Whether this interface should be protected by the firewall.',
 774         optional => 1,
 775     },
 776     link_down => {
 777         type => 'boolean',
 778         description => 'Whether this interface should be disconnected (like pulling the plug).',
 779         optional => 1,
 780     },
 781 };
 782
 783 my $netdesc = {
 784     optional => 1,
 785     type => 'string', format => $net_fmt,
 786     description => "Specify network devices.",
 787 };
 788
 789 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 790
 791 my $ipconfig_fmt = {
 792     ip => {
 793         type => 'string',
 794         format => 'pve-ipv4-config',
 795         format_description => 'IPv4Format/CIDR',
 796         description => 'IPv4 address in CIDR format.',
 797         optional => 1,
 798         default => 'dhcp',
 799     },
 800     gw => {
 801         type => 'string',
 802         format => 'ipv4',
 803         format_description => 'GatewayIPv4',
 804         description => 'Default gateway for IPv4 traffic.',
 805         optional => 1,
 806         requires => 'ip',
 807     },
 808     ip6 => {
 809         type => 'string',
 810         format => 'pve-ipv6-config',
 811         format_description => 'IPv6Format/CIDR',
 812         description => 'IPv6 address in CIDR format.',
 813         optional => 1,
 814         default => 'dhcp',
 815     },
 816     gw6 => {
 817         type => 'string',
 818         format => 'ipv6',
 819         format_description => 'GatewayIPv6',
 820         description => 'Default gateway for IPv6 traffic.',
 821         optional => 1,
 822         requires => 'ip6',
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 826 my $ipconfigdesc = {
 827     optional => 1,
 828     type => 'string', format => 'pve-qm-ipconfig',
 829     description => <<'EODESCR',
 830 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 831
 832 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 833
 834 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 835 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 836
 837 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 838 EODESCR
 839 };
 840 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 841
 842 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 843     $confdesc->{"net$i"} = $netdesc;
 844     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 845 }
 846
 847 foreach my $key (keys %$confdesc_cloudinit) {
 848     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 849 }
 850
 851 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 852 sub verify_volume_id_or_qm_path {
 853     my ($volid, $noerr) = @_;
 854
 855     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 856         return $volid;
 857     }
 858
 859     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 860     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 861     if ($@) {
 862         return undef if $noerr;
 863         die $@;
 864     }
 865     return $volid;
 866 }
 867
 868 my $drivename_hash;
 869
 870 my %drivedesc_base = (
 871     volume => { alias => 'file' },
 872     file => {
 873         type => 'string',
 874         format => 'pve-volume-id-or-qm-path',
 875         default_key => 1,
 876         format_description => 'volume',
 877         description => "The drive's backing volume.",
 878     },
 879     media => {
 880         type => 'string',
 881         enum => [qw(cdrom disk)],
 882         description => "The drive's media type.",
 883         default => 'disk',
 884         optional => 1
 885     },
 886     cyls => {
 887         type => 'integer',
 888         description => "Force the drive's physical geometry to have a specific cylinder count.",
 889         optional => 1
 890     },
 891     heads => {
 892         type => 'integer',
 893         description => "Force the drive's physical geometry to have a specific head count.",
 894         optional => 1
 895     },
 896     secs => {
 897         type => 'integer',
 898         description => "Force the drive's physical geometry to have a specific sector count.",
 899         optional => 1
 900     },
 901     trans => {
 902         type => 'string',
 903         enum => [qw(none lba auto)],
 904         description => "Force disk geometry bios translation mode.",
 905         optional => 1,
 906     },
 907     snapshot => {
 908         type => 'boolean',
 909         description => "Controls qemu's snapshot mode feature."
 910             . " If activated, changes made to the disk are temporary and will"
 911             . " be discarded when the VM is shutdown.",
 912         optional => 1,
 913     },
 914     cache => {
 915         type => 'string',
 916         enum => [qw(none writethrough writeback unsafe directsync)],
 917         description => "The drive's cache mode",
 918         optional => 1,
 919     },
 920     format => get_standard_option('pve-qm-image-format'),
 921     size => {
 922         type => 'string',
 923         format => 'disk-size',
 924         format_description => 'DiskSize',
 925         description => "Disk size. This is purely informational and has no effect.",
 926         optional => 1,
 927     },
 928     backup => {
 929         type => 'boolean',
 930         description => "Whether the drive should be included when making backups.",
 931         optional => 1,
 932     },
 933     replicate => {
 934         type => 'boolean',
 935         description => 'Whether the drive should considered for replication jobs.',
 936         optional => 1,
 937         default => 1,
 938     },
 939     rerror => {
 940         type => 'string',
 941         enum => [qw(ignore report stop)],
 942         description => 'Read error action.',
 943         optional => 1,
 944     },
 945     werror => {
 946         type => 'string',
 947         enum => [qw(enospc ignore report stop)],
 948         description => 'Write error action.',
 949         optional => 1,
 950     },
 951     aio => {
 952         type => 'string',
 953         enum => [qw(native threads)],
 954         description => 'AIO type to use.',
 955         optional => 1,
 956     },
 957     discard => {
 958         type => 'string',
 959         enum => [qw(ignore on)],
 960         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 961         optional => 1,
 962     },
 963     detect_zeroes => {
 964         type => 'boolean',
 965         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 966         optional => 1,
 967     },
 968     serial => {
 969         type => 'string',
 970         format => 'urlencoded',
 971         format_description => 'serial',
 972         maxLength => 20*3, # *3 since it's %xx url enoded
 973         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 974         optional => 1,
 975     },
 976     shared => {
 977         type => 'boolean',
 978         description => 'Mark this locally-managed volume as available on all nodes',
 979         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 980         optional => 1,
 981         default => 0,
 982     }
 983 );
 984
 985 my %iothread_fmt = ( iothread => {
 986         type => 'boolean',
 987         description => "Whether to use iothreads for this drive",
 988         optional => 1,
 989 });
 990
 991 my %model_fmt = (
 992     model => {
 993         type => 'string',
 994         format => 'urlencoded',
 995         format_description => 'model',
 996         maxLength => 40*3, # *3 since it's %xx url enoded
 997         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 998         optional => 1,
 999     },
1000 );
1001
1002 my %queues_fmt = (
1003     queues => {
1004         type => 'integer',
1005         description => "Number of queues.",
1006         minimum => 2,
1007         optional => 1
1008     }
1009 );
1010
1011 my %scsiblock_fmt = (
1012     scsiblock => {
1013         type => 'boolean',
1014         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1015         optional => 1,
1016         default => 0,
1017     },
1018 );
1019
1020 my %ssd_fmt = (
1021     ssd => {
1022         type => 'boolean',
1023         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1024         optional => 1,
1025     },
1026 );
1027
1028 my $add_throttle_desc = sub {
1029     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1030     my $d = {
1031         type => $type,
1032         format_description => $unit,
1033         description => "Maximum $what in $longunit.",
1034         optional => 1,
1035     };
1036     $d->{minimum} = $minimum if defined($minimum);
1037     $drivedesc_base{$key} = $d;
1038 };
1039 # throughput: (leaky bucket)
1040 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1041 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1042 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1043 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1044 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1045 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1046 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1047 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1048 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1049
1050 # pools: (pool of IO before throttling starts taking effect)
1051 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1052 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1053 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1054 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1055 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1056 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1057
1058 # burst lengths
1059 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1060 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1061 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1062 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1063 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1064 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1065
1066 # legacy support
1067 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1068 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1069 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1070 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1071
1072 my $ide_fmt = {
1073     %drivedesc_base,
1074     %model_fmt,
1075     %ssd_fmt,
1076 };
1077 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1078
1079 my $idedesc = {
1080     optional => 1,
1081     type => 'string', format => $ide_fmt,
1082     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1083 };
1084 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1085
1086 my $scsi_fmt = {
1087     %drivedesc_base,
1088     %iothread_fmt,
1089     %queues_fmt,
1090     %scsiblock_fmt,
1091     %ssd_fmt,
1092 };
1093 my $scsidesc = {
1094     optional => 1,
1095     type => 'string', format => $scsi_fmt,
1096     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1099
1100 my $sata_fmt = {
1101     %drivedesc_base,
1102     %ssd_fmt,
1103 };
1104 my $satadesc = {
1105     optional => 1,
1106     type => 'string', format => $sata_fmt,
1107     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1110
1111 my $virtio_fmt = {
1112     %drivedesc_base,
1113     %iothread_fmt,
1114 };
1115 my $virtiodesc = {
1116     optional => 1,
1117     type => 'string', format => $virtio_fmt,
1118     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1119 };
1120 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1121
1122 my $alldrive_fmt = {
1123     %drivedesc_base,
1124     %iothread_fmt,
1125     %model_fmt,
1126     %queues_fmt,
1127     %scsiblock_fmt,
1128     %ssd_fmt,
1129 };
1130
1131 my $efidisk_fmt = {
1132     volume => { alias => 'file' },
1133     file => {
1134         type => 'string',
1135         format => 'pve-volume-id-or-qm-path',
1136         default_key => 1,
1137         format_description => 'volume',
1138         description => "The drive's backing volume.",
1139     },
1140     format => get_standard_option('pve-qm-image-format'),
1141     size => {
1142         type => 'string',
1143         format => 'disk-size',
1144         format_description => 'DiskSize',
1145         description => "Disk size. This is purely informational and has no effect.",
1146         optional => 1,
1147     },
1148 };
1149
1150 my $efidisk_desc = {
1151     optional => 1,
1152     type => 'string', format => $efidisk_fmt,
1153     description => "Configure a Disk for storing EFI vars",
1154 };
1155
1156 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1157
1158 my $usb_fmt = {
1159     host => {
1160         default_key => 1,
1161         type => 'string', format => 'pve-qm-usb-device',
1162         format_description => 'HOSTUSBDEVICE|spice',
1163         description => <<EODESCR,
1164 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1165
1166  'bus-port(.port)*' (decimal numbers) or
1167  'vendor_id:product_id' (hexadeciaml numbers) or
1168  'spice'
1169
1170 You can use the 'lsusb -t' command to list existing usb devices.
1171
1172 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1173
1174 The value 'spice' can be used to add a usb redirection devices for spice.
1175 EODESCR
1176     },
1177     usb3 => {
1178         optional => 1,
1179         type => 'boolean',
1180         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1181         default => 0,
1182     },
1183 };
1184
1185 my $usbdesc = {
1186     optional => 1,
1187     type => 'string', format => $usb_fmt,
1188     description => "Configure an USB device (n is 0 to 4).",
1189 };
1190 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1191
1192 # NOTE: the match-groups of this regex are used in parse_hostpci
1193 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1194 my $hostpci_fmt = {
1195     host => {
1196         default_key => 1,
1197         type => 'string',
1198         pattern => qr/$PCIRE(;$PCIRE)*/,
1199         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1200         description => <<EODESCR,
1201 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1202 of PCI virtual functions of the host. HOSTPCIID syntax is:
1203
1204 'bus:dev.func' (hexadecimal numbers)
1205
1206 You can us the 'lspci' command to list existing PCI devices.
1207 EODESCR
1208     },
1209     rombar => {
1210         type => 'boolean',
1211         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1212         optional => 1,
1213         default => 1,
1214     },
1215     romfile => {
1216         type => 'string',
1217         pattern => '[^,;]+',
1218         format_description => 'string',
1219         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1220         optional => 1,
1221     },
1222     pcie => {
1223         type => 'boolean',
1224         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1225         optional => 1,
1226         default => 0,
1227     },
1228     'x-vga' => {
1229         type => 'boolean',
1230         description =>  "Enable vfio-vga device support.",
1231         optional => 1,
1232         default => 0,
1233     },
1234 };
1235 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1236
1237 my $hostpcidesc = {
1238         optional => 1,
1239         type => 'string', format => 'pve-qm-hostpci',
1240         description => "Map host PCI devices into guest.",
1241         verbose_description =>  <<EODESCR,
1242 Map host PCI devices into guest.
1243
1244 NOTE: This option allows direct access to host hardware. So it is no longer
1245 possible to migrate such machines - use with special care.
1246
1247 CAUTION: Experimental! User reported problems with this option.
1248 EODESCR
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1251
1252 my $serialdesc = {
1253         optional => 1,
1254         type => 'string',
1255         pattern => '(/dev/.+|socket)',
1256         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1257         verbose_description =>  <<EODESCR,
1258 Create a serial device inside the VM (n is 0 to 3), and pass through a
1259 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1260 host side (use 'qm terminal' to open a terminal connection).
1261
1262 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1263
1264 CAUTION: Experimental! User reported problems with this option.
1265 EODESCR
1266 };
1267
1268 my $paralleldesc= {
1269         optional => 1,
1270         type => 'string',
1271         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1272         description =>  "Map host parallel devices (n is 0 to 2).",
1273         verbose_description =>  <<EODESCR,
1274 Map host parallel devices (n is 0 to 2).
1275
1276 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1283     $confdesc->{"parallel$i"} = $paralleldesc;
1284 }
1285
1286 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1287     $confdesc->{"serial$i"} = $serialdesc;
1288 }
1289
1290 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1291     $confdesc->{"hostpci$i"} = $hostpcidesc;
1292 }
1293
1294 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1295     $drivename_hash->{"ide$i"} = 1;
1296     $confdesc->{"ide$i"} = $idedesc;
1297 }
1298
1299 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1300     $drivename_hash->{"sata$i"} = 1;
1301     $confdesc->{"sata$i"} = $satadesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1305     $drivename_hash->{"scsi$i"} = 1;
1306     $confdesc->{"scsi$i"} = $scsidesc ;
1307 }
1308
1309 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1310     $drivename_hash->{"virtio$i"} = 1;
1311     $confdesc->{"virtio$i"} = $virtiodesc;
1312 }
1313
1314 $drivename_hash->{efidisk0} = 1;
1315 $confdesc->{efidisk0} = $efidisk_desc;
1316
1317 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1318     $confdesc->{"usb$i"} = $usbdesc;
1319 }
1320
1321 my $unuseddesc = {
1322     optional => 1,
1323     type => 'string', format => 'pve-volume-id',
1324     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1325 };
1326
1327 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1328     $confdesc->{"unused$i"} = $unuseddesc;
1329 }
1330
1331 my $kvm_api_version = 0;
1332
1333 sub kvm_version {
1334
1335     return $kvm_api_version if $kvm_api_version;
1336
1337     my $fh = IO::File->new("</dev/kvm") ||
1338         return 0;
1339
1340     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1341         $kvm_api_version = $v;
1342     }
1343
1344     $fh->close();
1345
1346     return  $kvm_api_version;
1347 }
1348
1349 my $kvm_user_version;
1350
1351 sub kvm_user_version {
1352
1353     return $kvm_user_version if $kvm_user_version;
1354
1355     $kvm_user_version = 'unknown';
1356
1357     my $code = sub {
1358         my $line = shift;
1359         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1360             $kvm_user_version = $2;
1361         }
1362     };
1363
1364     eval { run_command("kvm -version", outfunc => $code); };
1365     warn $@ if $@;
1366
1367     return $kvm_user_version;
1368
1369 }
1370
1371 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1372
1373 sub valid_drive_names {
1374     # order is important - used to autoselect boot disk
1375     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1376             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1377             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1378             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1379             'efidisk0');
1380 }
1381
1382 sub is_valid_drivename {
1383     my $dev = shift;
1384
1385     return defined($drivename_hash->{$dev});
1386 }
1387
1388 sub option_exists {
1389     my $key = shift;
1390     return defined($confdesc->{$key});
1391 }
1392
1393 sub nic_models {
1394     return $nic_model_list;
1395 }
1396
1397 sub os_list_description {
1398
1399     return {
1400         other => 'Other',
1401         wxp => 'Windows XP',
1402         w2k => 'Windows 2000',
1403         w2k3 =>, 'Windows 2003',
1404         w2k8 => 'Windows 2008',
1405         wvista => 'Windows Vista',
1406         win7 => 'Windows 7',
1407         win8 => 'Windows 8/2012',
1408         win10 => 'Windows 10/2016',
1409         l24 => 'Linux 2.4',
1410         l26 => 'Linux 2.6',
1411     };
1412 }
1413
1414 my $cdrom_path;
1415
1416 sub get_cdrom_path {
1417
1418     return  $cdrom_path if $cdrom_path;
1419
1420     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1421     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1422     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1423 }
1424
1425 sub get_iso_path {
1426     my ($storecfg, $vmid, $cdrom) = @_;
1427
1428     if ($cdrom eq 'cdrom') {
1429         return get_cdrom_path();
1430     } elsif ($cdrom eq 'none') {
1431         return '';
1432     } elsif ($cdrom =~ m|^/|) {
1433         return $cdrom;
1434     } else {
1435         return PVE::Storage::path($storecfg, $cdrom);
1436     }
1437 }
1438
1439 # try to convert old style file names to volume IDs
1440 sub filename_to_volume_id {
1441     my ($vmid, $file, $media) = @_;
1442
1443      if (!($file eq 'none' || $file eq 'cdrom' ||
1444           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1445
1446         return undef if $file =~ m|/|;
1447
1448         if ($media && $media eq 'cdrom') {
1449             $file = "local:iso/$file";
1450         } else {
1451             $file = "local:$vmid/$file";
1452         }
1453     }
1454
1455     return $file;
1456 }
1457
1458 sub verify_media_type {
1459     my ($opt, $vtype, $media) = @_;
1460
1461     return if !$media;
1462
1463     my $etype;
1464     if ($media eq 'disk') {
1465         $etype = 'images';
1466     } elsif ($media eq 'cdrom') {
1467         $etype = 'iso';
1468     } else {
1469         die "internal error";
1470     }
1471
1472     return if ($vtype eq $etype);
1473
1474     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1475 }
1476
1477 sub cleanup_drive_path {
1478     my ($opt, $storecfg, $drive) = @_;
1479
1480     # try to convert filesystem paths to volume IDs
1481
1482     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1483         ($drive->{file} !~ m|^/dev/.+|) &&
1484         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1485         ($drive->{file} !~ m/^\d+$/)) {
1486         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1487         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1488         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1489         verify_media_type($opt, $vtype, $drive->{media});
1490         $drive->{file} = $volid;
1491     }
1492
1493     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1494 }
1495
1496 sub parse_hotplug_features {
1497     my ($data) = @_;
1498
1499     my $res = {};
1500
1501     return $res if $data eq '0';
1502
1503     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1504
1505     foreach my $feature (PVE::Tools::split_list($data)) {
1506         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1507             $res->{$1} = 1;
1508         } else {
1509             die "invalid hotplug feature '$feature'\n";
1510         }
1511     }
1512     return $res;
1513 }
1514
1515 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1516 sub pve_verify_hotplug_features {
1517     my ($value, $noerr) = @_;
1518
1519     return $value if parse_hotplug_features($value);
1520
1521     return undef if $noerr;
1522
1523     die "unable to parse hotplug option\n";
1524 }
1525
1526 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1527 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1528 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1529 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1530 #        [,iothread=on][,serial=serial][,model=model]
1531
1532 sub parse_drive {
1533     my ($key, $data) = @_;
1534
1535     my ($interface, $index);
1536
1537     if ($key =~ m/^([^\d]+)(\d+)$/) {
1538         $interface = $1;
1539         $index = $2;
1540     } else {
1541         return undef;
1542     }
1543
1544     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1545                                      : $confdesc->{$key}->{format};
1546     if (!$desc) {
1547         warn "invalid drive key: $key\n";
1548         return undef;
1549     }
1550     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1551     return undef if !$res;
1552     $res->{interface} = $interface;
1553     $res->{index} = $index;
1554
1555     my $error = 0;
1556     foreach my $opt (qw(bps bps_rd bps_wr)) {
1557         if (my $bps = defined(delete $res->{$opt})) {
1558             if (defined($res->{"m$opt"})) {
1559                 warn "both $opt and m$opt specified\n";
1560                 ++$error;
1561                 next;
1562             }
1563             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1564         }
1565     }
1566
1567     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1568     for my $requirement (
1569         [mbps_max => 'mbps'],
1570         [mbps_rd_max => 'mbps_rd'],
1571         [mbps_wr_max => 'mbps_wr'],
1572         [miops_max => 'miops'],
1573         [miops_rd_max => 'miops_rd'],
1574         [miops_wr_max => 'miops_wr'],
1575         [bps_max_length => 'mbps_max'],
1576         [bps_rd_max_length => 'mbps_rd_max'],
1577         [bps_wr_max_length => 'mbps_wr_max'],
1578         [iops_max_length => 'iops_max'],
1579         [iops_rd_max_length => 'iops_rd_max'],
1580         [iops_wr_max_length => 'iops_wr_max']) {
1581         my ($option, $requires) = @$requirement;
1582         if ($res->{$option} && !$res->{$requires}) {
1583             warn "$option requires $requires\n";
1584             ++$error;
1585         }
1586     }
1587
1588     return undef if $error;
1589
1590     return undef if $res->{mbps_rd} && $res->{mbps};
1591     return undef if $res->{mbps_wr} && $res->{mbps};
1592     return undef if $res->{iops_rd} && $res->{iops};
1593     return undef if $res->{iops_wr} && $res->{iops};
1594
1595     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1596         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1597         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1598         return undef if $res->{interface} eq 'virtio';
1599     }
1600
1601     if (my $size = $res->{size}) {
1602         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1603     }
1604
1605     return $res;
1606 }
1607
1608 sub print_drive {
1609     my ($vmid, $drive) = @_;
1610     my $data = { %$drive };
1611     delete $data->{$_} for qw(index interface);
1612     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1613 }
1614
1615 sub scsi_inquiry {
1616     my($fh, $noerr) = @_;
1617
1618     my $SG_IO = 0x2285;
1619     my $SG_GET_VERSION_NUM = 0x2282;
1620
1621     my $versionbuf = "\x00" x 8;
1622     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1623     if (!$ret) {
1624         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1625         return undef;
1626     }
1627     my $version = unpack("I", $versionbuf);
1628     if ($version < 30000) {
1629         die "scsi generic interface too old\n"  if !$noerr;
1630         return undef;
1631     }
1632
1633     my $buf = "\x00" x 36;
1634     my $sensebuf = "\x00" x 8;
1635     my $cmd = pack("C x3 C x1", 0x12, 36);
1636
1637     # see /usr/include/scsi/sg.h
1638     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1639
1640     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1641                       length($sensebuf), 0, length($buf), $buf,
1642                       $cmd, $sensebuf, 6000);
1643
1644     $ret = ioctl($fh, $SG_IO, $packet);
1645     if (!$ret) {
1646         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1647         return undef;
1648     }
1649
1650     my @res = unpack($sg_io_hdr_t, $packet);
1651     if ($res[17] || $res[18]) {
1652         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1653         return undef;
1654     }
1655
1656     my $res = {};
1657     (my $byte0, my $byte1, $res->{vendor},
1658      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1659
1660     $res->{removable} = $byte1 & 128 ? 1 : 0;
1661     $res->{type} = $byte0 & 31;
1662
1663     return $res;
1664 }
1665
1666 sub path_is_scsi {
1667     my ($path) = @_;
1668
1669     my $fh = IO::File->new("+<$path") || return undef;
1670     my $res = scsi_inquiry($fh, 1);
1671     close($fh);
1672
1673     return $res;
1674 }
1675
1676 sub machine_type_is_q35 {
1677     my ($conf) = @_;
1678
1679     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1680 }
1681
1682 sub print_tabletdevice_full {
1683     my ($conf) = @_;
1684
1685     my $q35 = machine_type_is_q35($conf);
1686
1687     # we use uhci for old VMs because tablet driver was buggy in older qemu
1688     my $usbbus = $q35 ? "ehci" : "uhci";
1689
1690     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1691 }
1692
1693 sub print_drivedevice_full {
1694     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1695
1696     my $device = '';
1697     my $maxdev = 0;
1698
1699     if ($drive->{interface} eq 'virtio') {
1700         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1701         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1702         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1703     } elsif ($drive->{interface} eq 'scsi') {
1704
1705         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1706         my $unit = $drive->{index} % $maxdev;
1707         my $devicetype = 'hd';
1708         my $path = '';
1709         if (drive_is_cdrom($drive)) {
1710             $devicetype = 'cd';
1711         } else {
1712             if ($drive->{file} =~ m|^/|) {
1713                 $path = $drive->{file};
1714                 if (my $info = path_is_scsi($path)) {
1715                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1716                         $devicetype = 'block';
1717                     } elsif ($info->{type} == 1) { # tape
1718                         $devicetype = 'generic';
1719                     }
1720                 }
1721             } else {
1722                  $path = PVE::Storage::path($storecfg, $drive->{file});
1723             }
1724
1725             if($path =~ m/^iscsi\:\/\//){
1726                 $devicetype = 'generic';
1727             }
1728         }
1729
1730         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1731             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1732         } else {
1733             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1734         }
1735
1736         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1737             $device .= ",rotation_rate=1";
1738         }
1739
1740     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1741         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1742         my $controller = int($drive->{index} / $maxdev);
1743         my $unit = $drive->{index} % $maxdev;
1744         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1745
1746         $device = "ide-$devicetype";
1747         if ($drive->{interface} eq 'ide') {
1748             $device .= ",bus=ide.$controller,unit=$unit";
1749         } else {
1750             $device .= ",bus=ahci$controller.$unit";
1751         }
1752         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1753
1754         if ($devicetype eq 'hd') {
1755             if (my $model = $drive->{model}) {
1756                 $model = URI::Escape::uri_unescape($model);
1757                 $device .= ",model=$model";
1758             }
1759             if ($drive->{ssd}) {
1760                 $device .= ",rotation_rate=1";
1761             }
1762         }
1763     } elsif ($drive->{interface} eq 'usb') {
1764         die "implement me";
1765         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1766     } else {
1767         die "unsupported interface type";
1768     }
1769
1770     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1771
1772     if (my $serial = $drive->{serial}) {
1773         $serial = URI::Escape::uri_unescape($serial);
1774         $device .= ",serial=$serial";
1775     }
1776
1777
1778     return $device;
1779 }
1780
1781 sub get_initiator_name {
1782     my $initiator;
1783
1784     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1785     while (defined(my $line = <$fh>)) {
1786         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1787         $initiator = $1;
1788         last;
1789     }
1790     $fh->close();
1791
1792     return $initiator;
1793 }
1794
1795 sub print_drive_full {
1796     my ($storecfg, $vmid, $drive) = @_;
1797
1798     my $path;
1799     my $volid = $drive->{file};
1800     my $format;
1801
1802     if (drive_is_cdrom($drive)) {
1803         $path = get_iso_path($storecfg, $vmid, $volid);
1804     } else {
1805         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1806         if ($storeid) {
1807             $path = PVE::Storage::path($storecfg, $volid);
1808             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1809             $format = qemu_img_format($scfg, $volname);
1810         } else {
1811             $path = $volid;
1812             $format = "raw";
1813         }
1814    }
1815
1816     my $opts = '';
1817     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1818     foreach my $o (@qemu_drive_options) {
1819         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1820     }
1821
1822     # snapshot only accepts on|off
1823     if (defined($drive->{snapshot})) {
1824         my $v = $drive->{snapshot} ? 'on' : 'off';
1825         $opts .= ",snapshot=$v";
1826     }
1827
1828     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1829         my ($dir, $qmpname) = @$type;
1830         if (my $v = $drive->{"mbps$dir"}) {
1831             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1832         }
1833         if (my $v = $drive->{"mbps${dir}_max"}) {
1834             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1835         }
1836         if (my $v = $drive->{"bps${dir}_max_length"}) {
1837             $opts .= ",throttling.bps$qmpname-max-length=$v";
1838         }
1839         if (my $v = $drive->{"iops${dir}"}) {
1840             $opts .= ",throttling.iops$qmpname=$v";
1841         }
1842         if (my $v = $drive->{"iops${dir}_max"}) {
1843             $opts .= ",throttling.iops$qmpname-max=$v";
1844         }
1845         if (my $v = $drive->{"iops${dir}_max_length"}) {
1846             $opts .= ",throttling.iops$qmpname-max-length=$v";
1847         }
1848     }
1849
1850     $opts .= ",format=$format" if $format && !$drive->{format};
1851
1852     my $cache_direct = 0;
1853
1854     if (my $cache = $drive->{cache}) {
1855         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1856     } elsif (!drive_is_cdrom($drive)) {
1857         $opts .= ",cache=none";
1858         $cache_direct = 1;
1859     }
1860
1861     # aio native works only with O_DIRECT
1862     if (!$drive->{aio}) {
1863         if($cache_direct) {
1864             $opts .= ",aio=native";
1865         } else {
1866             $opts .= ",aio=threads";
1867         }
1868     }
1869
1870     if (!drive_is_cdrom($drive)) {
1871         my $detectzeroes;
1872         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1873             $detectzeroes = 'off';
1874         } elsif ($drive->{discard}) {
1875             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1876         } else {
1877             # This used to be our default with discard not being specified:
1878             $detectzeroes = 'on';
1879         }
1880         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1881     }
1882
1883     my $pathinfo = $path ? "file=$path," : '';
1884
1885     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1886 }
1887
1888 sub print_netdevice_full {
1889     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1890
1891     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1892
1893     my $device = $net->{model};
1894     if ($net->{model} eq 'virtio') {
1895          $device = 'virtio-net-pci';
1896      };
1897
1898     my $pciaddr = print_pci_addr("$netid", $bridges);
1899     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1900     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1901         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1902         my $vectors = $net->{queues} * 2 + 2;
1903         $tmpstr .= ",vectors=$vectors,mq=on";
1904     }
1905     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1906
1907     if ($use_old_bios_files) {
1908         my $romfile;
1909         if ($device eq 'virtio-net-pci') {
1910             $romfile = 'pxe-virtio.rom';
1911         } elsif ($device eq 'e1000') {
1912             $romfile = 'pxe-e1000.rom';
1913         } elsif ($device eq 'ne2k') {
1914             $romfile = 'pxe-ne2k_pci.rom';
1915         } elsif ($device eq 'pcnet') {
1916             $romfile = 'pxe-pcnet.rom';
1917         } elsif ($device eq 'rtl8139') {
1918             $romfile = 'pxe-rtl8139.rom';
1919         }
1920         $tmpstr .= ",romfile=$romfile" if $romfile;
1921     }
1922
1923     return $tmpstr;
1924 }
1925
1926 sub print_netdev_full {
1927     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1928
1929     my $i = '';
1930     if ($netid =~ m/^net(\d+)$/) {
1931         $i = int($1);
1932     }
1933
1934     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1935
1936     my $ifname = "tap${vmid}i$i";
1937
1938     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1939     die "interface name '$ifname' is too long (max 15 character)\n"
1940         if length($ifname) >= 16;
1941
1942     my $vhostparam = '';
1943     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1944
1945     my $vmname = $conf->{name} || "vm$vmid";
1946
1947     my $netdev = "";
1948     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1949
1950     if ($net->{bridge}) {
1951         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1952     } else {
1953         $netdev = "type=user,id=$netid,hostname=$vmname";
1954     }
1955
1956     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1957
1958     return $netdev;
1959 }
1960
1961
1962 sub print_cpu_device {
1963     my ($conf, $id) = @_;
1964
1965     my $kvm = $conf->{kvm} // 1;
1966     my $cpu = $kvm ? "kvm64" : "qemu64";
1967     if (my $cputype = $conf->{cpu}) {
1968         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1969             or die "Cannot parse cpu description: $cputype\n";
1970         $cpu = $cpuconf->{cputype};
1971     }
1972
1973     my $cores = $conf->{cores} || 1;
1974
1975     my $current_core = ($id - 1) % $cores;
1976     my $current_socket = int(($id - 1 - $current_core)/$cores);
1977
1978     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1979 }
1980
1981 my $vga_map = {
1982     'cirrus' => 'cirrus-vga',
1983     'std' => 'VGA',
1984     'vmware' => 'vmware-svga',
1985     'virtio' => 'virtio-vga',
1986 };
1987
1988 sub print_vga_device {
1989     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
1990
1991     my $type = $vga_map->{$vga->{type}};
1992     my $vgamem_mb = $vga->{memory};
1993     if ($qxlnum) {
1994         $type = $id ? 'qxl' : 'qxl-vga';
1995     }
1996     die "no devicetype for $vga->{type}\n" if !$type;
1997
1998     my $memory = "";
1999     if ($vgamem_mb) {
2000         if ($vga->{type} eq 'virtio') {
2001             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2002             $memory = ",max_hostmem=$bytes";
2003         } elsif ($qxlnum) {
2004             # from https://www.spice-space.org/multiple-monitors.html
2005             $memory = ",vgamem_mb=$vga->{memory}";
2006             my $ram = $vgamem_mb * 4;
2007             my $vram = $vgamem_mb * 2;
2008             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2009         } else {
2010             $memory = ",vgamem_mb=$vga->{memory}";
2011         }
2012     } elsif ($qxlnum && $id) {
2013         $memory = ",ram_size=67108864,vram_size=33554432";
2014     }
2015
2016     my $q35 = machine_type_is_q35($conf);
2017     my $vgaid = "vga" . ($id // '');
2018     my $pciaddr;
2019
2020     if ($q35 && $vgaid eq 'vga') {
2021         # the first display uses pcie.0 bus on q35 machines
2022         $pciaddr = print_pcie_addr($vgaid, $bridges);
2023     } else {
2024         $pciaddr = print_pci_addr($vgaid, $bridges);
2025     }
2026
2027     return "$type,id=${vgaid}${memory}${pciaddr}";
2028 }
2029
2030 sub drive_is_cloudinit {
2031     my ($drive) = @_;
2032     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2033 }
2034
2035 sub drive_is_cdrom {
2036     my ($drive, $exclude_cloudinit) = @_;
2037
2038     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2039
2040     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2041
2042 }
2043
2044 sub parse_number_sets {
2045     my ($set) = @_;
2046     my $res = [];
2047     foreach my $part (split(/;/, $set)) {
2048         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2049             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2050             push @$res, [ $1, $2 ];
2051         } else {
2052             die "invalid range: $part\n";
2053         }
2054     }
2055     return $res;
2056 }
2057
2058 sub parse_numa {
2059     my ($data) = @_;
2060
2061     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2062     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2063     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2064     return $res;
2065 }
2066
2067 sub parse_hostpci {
2068     my ($value) = @_;
2069
2070     return undef if !$value;
2071
2072     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2073
2074     my @idlist = split(/;/, $res->{host});
2075     delete $res->{host};
2076     foreach my $id (@idlist) {
2077         if ($id =~ /^$PCIRE$/) {
2078             if (defined($2)) {
2079                 push @{$res->{pciid}}, { id => $1, function => $2 };
2080             } else {
2081                 my $pcidevices = lspci($1);
2082                 $res->{pciid} = $pcidevices->{$1};
2083             }
2084         } else {
2085             # should have been caught by parse_property_string already
2086             die "failed to parse PCI id: $id\n";
2087         }
2088     }
2089     return $res;
2090 }
2091
2092 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2093 sub parse_net {
2094     my ($data) = @_;
2095
2096     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2097     if ($@) {
2098         warn $@;
2099         return undef;
2100     }
2101     if (!defined($res->{macaddr})) {
2102         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2103         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2104     }
2105     return $res;
2106 }
2107
2108 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2109 sub parse_ipconfig {
2110     my ($data) = @_;
2111
2112     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2113     if ($@) {
2114         warn $@;
2115         return undef;
2116     }
2117
2118     if ($res->{gw} && !$res->{ip}) {
2119         warn 'gateway specified without specifying an IP address';
2120         return undef;
2121     }
2122     if ($res->{gw6} && !$res->{ip6}) {
2123         warn 'IPv6 gateway specified without specifying an IPv6 address';
2124         return undef;
2125     }
2126     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2127         warn 'gateway specified together with DHCP';
2128         return undef;
2129     }
2130     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2131         # gw6 + auto/dhcp
2132         warn "IPv6 gateway specified together with $res->{ip6} address";
2133         return undef;
2134     }
2135
2136     if (!$res->{ip} && !$res->{ip6}) {
2137         return { ip => 'dhcp', ip6 => 'dhcp' };
2138     }
2139
2140     return $res;
2141 }
2142
2143 sub print_net {
2144     my $net = shift;
2145
2146     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2147 }
2148
2149 sub add_random_macs {
2150     my ($settings) = @_;
2151
2152     foreach my $opt (keys %$settings) {
2153         next if $opt !~ m/^net(\d+)$/;
2154         my $net = parse_net($settings->{$opt});
2155         next if !$net;
2156         $settings->{$opt} = print_net($net);
2157     }
2158 }
2159
2160 sub vm_is_volid_owner {
2161     my ($storecfg, $vmid, $volid) = @_;
2162
2163     if ($volid !~  m|^/|) {
2164         my ($path, $owner);
2165         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2166         if ($owner && ($owner == $vmid)) {
2167             return 1;
2168         }
2169     }
2170
2171     return undef;
2172 }
2173
2174 sub split_flagged_list {
2175     my $text = shift || '';
2176     $text =~ s/[,;]/ /g;
2177     $text =~ s/^\s+//;
2178     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2179 }
2180
2181 sub join_flagged_list {
2182     my ($how, $lst) = @_;
2183     join $how, map { $lst->{$_} . $_ } keys %$lst;
2184 }
2185
2186 sub vmconfig_delete_pending_option {
2187     my ($conf, $key, $force) = @_;
2188
2189     delete $conf->{pending}->{$key};
2190     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2191     $pending_delete_hash->{$key} = $force ? '!' : '';
2192     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2193 }
2194
2195 sub vmconfig_undelete_pending_option {
2196     my ($conf, $key) = @_;
2197
2198     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2199     delete $pending_delete_hash->{$key};
2200
2201     if (%$pending_delete_hash) {
2202         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2203     } else {
2204         delete $conf->{pending}->{delete};
2205     }
2206 }
2207
2208 sub vmconfig_register_unused_drive {
2209     my ($storecfg, $vmid, $conf, $drive) = @_;
2210
2211     if (drive_is_cloudinit($drive)) {
2212         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2213         warn $@ if $@;
2214     } elsif (!drive_is_cdrom($drive)) {
2215         my $volid = $drive->{file};
2216         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2217             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2218         }
2219     }
2220 }
2221
2222 sub vmconfig_cleanup_pending {
2223     my ($conf) = @_;
2224
2225     # remove pending changes when nothing changed
2226     my $changes;
2227     foreach my $opt (keys %{$conf->{pending}}) {
2228         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2229             $changes = 1;
2230             delete $conf->{pending}->{$opt};
2231         }
2232     }
2233
2234     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2235     my $pending_delete_hash = {};
2236     while (my ($opt, $force) = each %$current_delete_hash) {
2237         if (defined($conf->{$opt})) {
2238             $pending_delete_hash->{$opt} = $force;
2239         } else {
2240             $changes = 1;
2241         }
2242     }
2243
2244     if (%$pending_delete_hash) {
2245         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2246     } else {
2247         delete $conf->{pending}->{delete};
2248     }
2249
2250     return $changes;
2251 }
2252
2253 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2254 my $smbios1_fmt = {
2255     uuid => {
2256         type => 'string',
2257         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2258         format_description => 'UUID',
2259         description => "Set SMBIOS1 UUID.",
2260         optional => 1,
2261     },
2262     version => {
2263         type => 'string',
2264         pattern => '\S+',
2265         format_description => 'string',
2266         description => "Set SMBIOS1 version.",
2267         optional => 1,
2268     },
2269     serial => {
2270         type => 'string',
2271         pattern => '\S+',
2272         format_description => 'string',
2273         description => "Set SMBIOS1 serial number.",
2274         optional => 1,
2275     },
2276     manufacturer => {
2277         type => 'string',
2278         pattern => '\S+',
2279         format_description => 'string',
2280         description => "Set SMBIOS1 manufacturer.",
2281         optional => 1,
2282     },
2283     product => {
2284         type => 'string',
2285         pattern => '\S+',
2286         format_description => 'string',
2287         description => "Set SMBIOS1 product ID.",
2288         optional => 1,
2289     },
2290     sku => {
2291         type => 'string',
2292         pattern => '\S+',
2293         format_description => 'string',
2294         description => "Set SMBIOS1 SKU string.",
2295         optional => 1,
2296     },
2297     family => {
2298         type => 'string',
2299         pattern => '\S+',
2300         format_description => 'string',
2301         description => "Set SMBIOS1 family string.",
2302         optional => 1,
2303     },
2304 };
2305
2306 sub parse_smbios1 {
2307     my ($data) = @_;
2308
2309     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2310     warn $@ if $@;
2311     return $res;
2312 }
2313
2314 sub print_smbios1 {
2315     my ($smbios1) = @_;
2316     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2317 }
2318
2319 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2320
2321 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2322 sub verify_bootdisk {
2323     my ($value, $noerr) = @_;
2324
2325     return $value if is_valid_drivename($value);
2326
2327     return undef if $noerr;
2328
2329     die "invalid boot disk '$value'\n";
2330 }
2331
2332 sub parse_watchdog {
2333     my ($value) = @_;
2334
2335     return undef if !$value;
2336
2337     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2338     warn $@ if $@;
2339     return $res;
2340 }
2341
2342 sub parse_guest_agent {
2343     my ($value) = @_;
2344
2345     return {} if !defined($value->{agent});
2346
2347     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2348     warn $@ if $@;
2349
2350     # if the agent is disabled ignore the other potentially set properties
2351     return {} if !$res->{enabled};
2352     return $res;
2353 }
2354
2355 sub parse_vga {
2356     my ($value) = @_;
2357
2358     return {} if !$value;
2359     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2360     warn $@ if $@;
2361     return $res;
2362 }
2363
2364 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2365 sub verify_usb_device {
2366     my ($value, $noerr) = @_;
2367
2368     return $value if parse_usb_device($value);
2369
2370     return undef if $noerr;
2371
2372     die "unable to parse usb device\n";
2373 }
2374
2375 # add JSON properties for create and set function
2376 sub json_config_properties {
2377     my $prop = shift;
2378
2379     foreach my $opt (keys %$confdesc) {
2380         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2381         $prop->{$opt} = $confdesc->{$opt};
2382     }
2383
2384     return $prop;
2385 }
2386
2387 # return copy of $confdesc_cloudinit to generate documentation
2388 sub cloudinit_config_properties {
2389
2390     return dclone($confdesc_cloudinit);
2391 }
2392
2393 sub check_type {
2394     my ($key, $value) = @_;
2395
2396     die "unknown setting '$key'\n" if !$confdesc->{$key};
2397
2398     my $type = $confdesc->{$key}->{type};
2399
2400     if (!defined($value)) {
2401         die "got undefined value\n";
2402     }
2403
2404     if ($value =~ m/[\n\r]/) {
2405         die "property contains a line feed\n";
2406     }
2407
2408     if ($type eq 'boolean') {
2409         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2410         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2411         die "type check ('boolean') failed - got '$value'\n";
2412     } elsif ($type eq 'integer') {
2413         return int($1) if $value =~ m/^(\d+)$/;
2414         die "type check ('integer') failed - got '$value'\n";
2415     } elsif ($type eq 'number') {
2416         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2417         die "type check ('number') failed - got '$value'\n";
2418     } elsif ($type eq 'string') {
2419         if (my $fmt = $confdesc->{$key}->{format}) {
2420             PVE::JSONSchema::check_format($fmt, $value);
2421             return $value;
2422         }
2423         $value =~ s/^\"(.*)\"$/$1/;
2424         return $value;
2425     } else {
2426         die "internal error"
2427     }
2428 }
2429
2430 sub check_iommu_support{
2431     #fixme : need to check IOMMU support
2432     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2433
2434     my $iommu=1;
2435     return $iommu;
2436
2437 }
2438
2439 sub touch_config {
2440     my ($vmid) = @_;
2441
2442     my $conf = PVE::QemuConfig->config_file($vmid);
2443     utime undef, undef, $conf;
2444 }
2445
2446 sub destroy_vm {
2447     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2448
2449     my $conffile = PVE::QemuConfig->config_file($vmid);
2450
2451     my $conf = PVE::QemuConfig->load_config($vmid);
2452
2453     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2454
2455     if ($conf->{template}) {
2456         # check if any base image is still used by a linked clone
2457         foreach_drive($conf, sub {
2458                 my ($ds, $drive) = @_;
2459
2460                 return if drive_is_cdrom($drive);
2461
2462                 my $volid = $drive->{file};
2463
2464                 return if !$volid || $volid =~ m|^/|;
2465
2466                 die "base volume '$volid' is still in use by linked cloned\n"
2467                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2468
2469         });
2470     }
2471
2472     # only remove disks owned by this VM
2473     foreach_drive($conf, sub {
2474         my ($ds, $drive) = @_;
2475
2476         return if drive_is_cdrom($drive, 1);
2477
2478         my $volid = $drive->{file};
2479
2480         return if !$volid || $volid =~ m|^/|;
2481
2482         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2483         return if !$path || !$owner || ($owner != $vmid);
2484
2485         eval {
2486             PVE::Storage::vdisk_free($storecfg, $volid);
2487         };
2488         warn "Could not remove disk '$volid', check manually: $@" if $@;
2489
2490     });
2491
2492     if ($keep_empty_config) {
2493         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2494     } else {
2495         unlink $conffile;
2496     }
2497
2498     # also remove unused disk
2499     eval {
2500         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2501
2502         eval {
2503             PVE::Storage::foreach_volid($dl, sub {
2504                 my ($volid, $sid, $volname, $d) = @_;
2505                 PVE::Storage::vdisk_free($storecfg, $volid);
2506             });
2507         };
2508         warn $@ if $@;
2509
2510     };
2511     warn $@ if $@;
2512 }
2513
2514 sub parse_vm_config {
2515     my ($filename, $raw) = @_;
2516
2517     return undef if !defined($raw);
2518
2519     my $res = {
2520         digest => Digest::SHA::sha1_hex($raw),
2521         snapshots => {},
2522         pending => {},
2523     };
2524
2525     $filename =~ m|/qemu-server/(\d+)\.conf$|
2526         || die "got strange filename '$filename'";
2527
2528     my $vmid = $1;
2529
2530     my $conf = $res;
2531     my $descr;
2532     my $section = '';
2533
2534     my @lines = split(/\n/, $raw);
2535     foreach my $line (@lines) {
2536         next if $line =~ m/^\s*$/;
2537
2538         if ($line =~ m/^\[PENDING\]\s*$/i) {
2539             $section = 'pending';
2540             if (defined($descr)) {
2541                 $descr =~ s/\s+$//;
2542                 $conf->{description} = $descr;
2543             }
2544             $descr = undef;
2545             $conf = $res->{$section} = {};
2546             next;
2547
2548         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2549             $section = $1;
2550             if (defined($descr)) {
2551                 $descr =~ s/\s+$//;
2552                 $conf->{description} = $descr;
2553             }
2554             $descr = undef;
2555             $conf = $res->{snapshots}->{$section} = {};
2556             next;
2557         }
2558
2559         if ($line =~ m/^\#(.*)\s*$/) {
2560             $descr = '' if !defined($descr);
2561             $descr .= PVE::Tools::decode_text($1) . "\n";
2562             next;
2563         }
2564
2565         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2566             $descr = '' if !defined($descr);
2567             $descr .= PVE::Tools::decode_text($2);
2568         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2569             $conf->{snapstate} = $1;
2570         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2571             my $key = $1;
2572             my $value = $2;
2573             $conf->{$key} = $value;
2574         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2575             my $value = $1;
2576             if ($section eq 'pending') {
2577                 $conf->{delete} = $value; # we parse this later
2578             } else {
2579                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2580             }
2581         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2582             my $key = $1;
2583             my $value = $2;
2584             eval { $value = check_type($key, $value); };
2585             if ($@) {
2586                 warn "vm $vmid - unable to parse value of '$key' - $@";
2587             } else {
2588                 $key = 'ide2' if $key eq 'cdrom';
2589                 my $fmt = $confdesc->{$key}->{format};
2590                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2591                     my $v = parse_drive($key, $value);
2592                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2593                         $v->{file} = $volid;
2594                         $value = print_drive($vmid, $v);
2595                     } else {
2596                         warn "vm $vmid - unable to parse value of '$key'\n";
2597                         next;
2598                     }
2599                 }
2600
2601                 $conf->{$key} = $value;
2602             }
2603         }
2604     }
2605
2606     if (defined($descr)) {
2607         $descr =~ s/\s+$//;
2608         $conf->{description} = $descr;
2609     }
2610     delete $res->{snapstate}; # just to be sure
2611
2612     return $res;
2613 }
2614
2615 sub write_vm_config {
2616     my ($filename, $conf) = @_;
2617
2618     delete $conf->{snapstate}; # just to be sure
2619
2620     if ($conf->{cdrom}) {
2621         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2622         $conf->{ide2} = $conf->{cdrom};
2623         delete $conf->{cdrom};
2624     }
2625
2626     # we do not use 'smp' any longer
2627     if ($conf->{sockets}) {
2628         delete $conf->{smp};
2629     } elsif ($conf->{smp}) {
2630         $conf->{sockets} = $conf->{smp};
2631         delete $conf->{cores};
2632         delete $conf->{smp};
2633     }
2634
2635     my $used_volids = {};
2636
2637     my $cleanup_config = sub {
2638         my ($cref, $pending, $snapname) = @_;
2639
2640         foreach my $key (keys %$cref) {
2641             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2642                 $key eq 'snapstate' || $key eq 'pending';
2643             my $value = $cref->{$key};
2644             if ($key eq 'delete') {
2645                 die "propertry 'delete' is only allowed in [PENDING]\n"
2646                     if !$pending;
2647                 # fixme: check syntax?
2648                 next;
2649             }
2650             eval { $value = check_type($key, $value); };
2651             die "unable to parse value of '$key' - $@" if $@;
2652
2653             $cref->{$key} = $value;
2654
2655             if (!$snapname && is_valid_drivename($key)) {
2656                 my $drive = parse_drive($key, $value);
2657                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2658             }
2659         }
2660     };
2661
2662     &$cleanup_config($conf);
2663
2664     &$cleanup_config($conf->{pending}, 1);
2665
2666     foreach my $snapname (keys %{$conf->{snapshots}}) {
2667         die "internal error" if $snapname eq 'pending';
2668         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2669     }
2670
2671     # remove 'unusedX' settings if we re-add a volume
2672     foreach my $key (keys %$conf) {
2673         my $value = $conf->{$key};
2674         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2675             delete $conf->{$key};
2676         }
2677     }
2678
2679     my $generate_raw_config = sub {
2680         my ($conf, $pending) = @_;
2681
2682         my $raw = '';
2683
2684         # add description as comment to top of file
2685         if (defined(my $descr = $conf->{description})) {
2686             if ($descr) {
2687                 foreach my $cl (split(/\n/, $descr)) {
2688                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2689                 }
2690             } else {
2691                 $raw .= "#\n" if $pending;
2692             }
2693         }
2694
2695         foreach my $key (sort keys %$conf) {
2696             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2697             $raw .= "$key: $conf->{$key}\n";
2698         }
2699         return $raw;
2700     };
2701
2702     my $raw = &$generate_raw_config($conf);
2703
2704     if (scalar(keys %{$conf->{pending}})){
2705         $raw .= "\n[PENDING]\n";
2706         $raw .= &$generate_raw_config($conf->{pending}, 1);
2707     }
2708
2709     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2710         $raw .= "\n[$snapname]\n";
2711         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2712     }
2713
2714     return $raw;
2715 }
2716
2717 sub load_defaults {
2718
2719     my $res = {};
2720
2721     # we use static defaults from our JSON schema configuration
2722     foreach my $key (keys %$confdesc) {
2723         if (defined(my $default = $confdesc->{$key}->{default})) {
2724             $res->{$key} = $default;
2725         }
2726     }
2727
2728     return $res;
2729 }
2730
2731 sub config_list {
2732     my $vmlist = PVE::Cluster::get_vmlist();
2733     my $res = {};
2734     return $res if !$vmlist || !$vmlist->{ids};
2735     my $ids = $vmlist->{ids};
2736
2737     foreach my $vmid (keys %$ids) {
2738         my $d = $ids->{$vmid};
2739         next if !$d->{node} || $d->{node} ne $nodename;
2740         next if !$d->{type} || $d->{type} ne 'qemu';
2741         $res->{$vmid}->{exists} = 1;
2742     }
2743     return $res;
2744 }
2745
2746 # test if VM uses local resources (to prevent migration)
2747 sub check_local_resources {
2748     my ($conf, $noerr) = @_;
2749
2750     my $loc_res = 0;
2751
2752     $loc_res = 1 if $conf->{hostusb}; # old syntax
2753     $loc_res = 1 if $conf->{hostpci}; # old syntax
2754
2755     foreach my $k (keys %$conf) {
2756         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2757         # sockets are safe: they will recreated be on the target side post-migrate
2758         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2759         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2760     }
2761
2762     die "VM uses local resources\n" if $loc_res && !$noerr;
2763
2764     return $loc_res;
2765 }
2766
2767 # check if used storages are available on all nodes (use by migrate)
2768 sub check_storage_availability {
2769     my ($storecfg, $conf, $node) = @_;
2770
2771     foreach_drive($conf, sub {
2772         my ($ds, $drive) = @_;
2773
2774         my $volid = $drive->{file};
2775         return if !$volid;
2776
2777         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2778         return if !$sid;
2779
2780         # check if storage is available on both nodes
2781         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2782         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2783    });
2784 }
2785
2786 # list nodes where all VM images are available (used by has_feature API)
2787 sub shared_nodes {
2788     my ($conf, $storecfg) = @_;
2789
2790     my $nodelist = PVE::Cluster::get_nodelist();
2791     my $nodehash = { map { $_ => 1 } @$nodelist };
2792     my $nodename = PVE::INotify::nodename();
2793
2794     foreach_drive($conf, sub {
2795         my ($ds, $drive) = @_;
2796
2797         my $volid = $drive->{file};
2798         return if !$volid;
2799
2800         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2801         if ($storeid) {
2802             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2803             if ($scfg->{disable}) {
2804                 $nodehash = {};
2805             } elsif (my $avail = $scfg->{nodes}) {
2806                 foreach my $node (keys %$nodehash) {
2807                     delete $nodehash->{$node} if !$avail->{$node};
2808                 }
2809             } elsif (!$scfg->{shared}) {
2810                 foreach my $node (keys %$nodehash) {
2811                     delete $nodehash->{$node} if $node ne $nodename
2812                 }
2813             }
2814         }
2815     });
2816
2817     return $nodehash
2818 }
2819
2820 sub check_cmdline {
2821     my ($pidfile, $pid) = @_;
2822
2823     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2824     if (defined($fh)) {
2825         my $line = <$fh>;
2826         $fh->close;
2827         return undef if !$line;
2828         my @param = split(/\0/, $line);
2829
2830         my $cmd = $param[0];
2831         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2832
2833         for (my $i = 0; $i < scalar (@param); $i++) {
2834             my $p = $param[$i];
2835             next if !$p;
2836             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2837                 my $p = $param[$i+1];
2838                 return 1 if $p && ($p eq $pidfile);
2839                 return undef;
2840             }
2841         }
2842     }
2843     return undef;
2844 }
2845
2846 sub check_running {
2847     my ($vmid, $nocheck, $node) = @_;
2848
2849     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2850
2851     die "unable to find configuration file for VM $vmid - no such machine\n"
2852         if !$nocheck && ! -f $filename;
2853
2854     my $pidfile = pidfile_name($vmid);
2855
2856     if (my $fd = IO::File->new("<$pidfile")) {
2857         my $st = stat($fd);
2858         my $line = <$fd>;
2859         close($fd);
2860
2861         my $mtime = $st->mtime;
2862         if ($mtime > time()) {
2863             warn "file '$filename' modified in future\n";
2864         }
2865
2866         if ($line =~ m/^(\d+)$/) {
2867             my $pid = $1;
2868             if (check_cmdline($pidfile, $pid)) {
2869                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2870                     return $pid;
2871                 }
2872             }
2873         }
2874     }
2875
2876     return undef;
2877 }
2878
2879 sub vzlist {
2880
2881     my $vzlist = config_list();
2882
2883     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2884
2885     while (defined(my $de = $fd->read)) {
2886         next if $de !~ m/^(\d+)\.pid$/;
2887         my $vmid = $1;
2888         next if !defined($vzlist->{$vmid});
2889         if (my $pid = check_running($vmid)) {
2890             $vzlist->{$vmid}->{pid} = $pid;
2891         }
2892     }
2893
2894     return $vzlist;
2895 }
2896
2897 sub disksize {
2898     my ($storecfg, $conf) = @_;
2899
2900     my $bootdisk = $conf->{bootdisk};
2901     return undef if !$bootdisk;
2902     return undef if !is_valid_drivename($bootdisk);
2903
2904     return undef if !$conf->{$bootdisk};
2905
2906     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2907     return undef if !defined($drive);
2908
2909     return undef if drive_is_cdrom($drive);
2910
2911     my $volid = $drive->{file};
2912     return undef if !$volid;
2913
2914     return $drive->{size};
2915 }
2916
2917 our $vmstatus_return_properties = {
2918     vmid => get_standard_option('pve-vmid'),
2919     status => {
2920         description => "Qemu process status.",
2921         type => 'string',
2922         enum => ['stopped', 'running'],
2923     },
2924     maxmem => {
2925         description => "Maximum memory in bytes.",
2926         type => 'integer',
2927         optional => 1,
2928         renderer => 'bytes',
2929     },
2930     maxdisk => {
2931         description => "Root disk size in bytes.",
2932         type => 'integer',
2933         optional => 1,
2934         renderer => 'bytes',
2935     },
2936     name => {
2937         description => "VM name.",
2938         type => 'string',
2939         optional => 1,
2940     },
2941     qmpstatus => {
2942         description => "Qemu QMP agent status.",
2943         type => 'string',
2944         optional => 1,
2945     },
2946     pid => {
2947         description => "PID of running qemu process.",
2948         type => 'integer',
2949         optional => 1,
2950     },
2951     uptime => {
2952         description => "Uptime.",
2953         type => 'integer',
2954         optional => 1,
2955         renderer => 'duration',
2956     },
2957     cpus => {
2958         description => "Maximum usable CPUs.",
2959         type => 'number',
2960         optional => 1,
2961     },
2962 };
2963
2964 my $last_proc_pid_stat;
2965
2966 # get VM status information
2967 # This must be fast and should not block ($full == false)
2968 # We only query KVM using QMP if $full == true (this can be slow)
2969 sub vmstatus {
2970     my ($opt_vmid, $full) = @_;
2971
2972     my $res = {};
2973
2974     my $storecfg = PVE::Storage::config();
2975
2976     my $list = vzlist();
2977     my $defaults = load_defaults();
2978
2979     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2980
2981     my $cpucount = $cpuinfo->{cpus} || 1;
2982
2983     foreach my $vmid (keys %$list) {
2984         next if $opt_vmid && ($vmid ne $opt_vmid);
2985
2986         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2987         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2988
2989         my $d = { vmid => $vmid };
2990         $d->{pid} = $list->{$vmid}->{pid};
2991
2992         # fixme: better status?
2993         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2994
2995         my $size = disksize($storecfg, $conf);
2996         if (defined($size)) {
2997             $d->{disk} = 0; # no info available
2998             $d->{maxdisk} = $size;
2999         } else {
3000             $d->{disk} = 0;
3001             $d->{maxdisk} = 0;
3002         }
3003
3004         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3005             * ($conf->{cores} || $defaults->{cores});
3006         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3007         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3008
3009         $d->{name} = $conf->{name} || "VM $vmid";
3010         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3011             : $defaults->{memory}*(1024*1024);
3012
3013         if ($conf->{balloon}) {
3014             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3015             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3016                 : $defaults->{shares};
3017         }
3018
3019         $d->{uptime} = 0;
3020         $d->{cpu} = 0;
3021         $d->{mem} = 0;
3022
3023         $d->{netout} = 0;
3024         $d->{netin} = 0;
3025
3026         $d->{diskread} = 0;
3027         $d->{diskwrite} = 0;
3028
3029         $d->{template} = PVE::QemuConfig->is_template($conf);
3030
3031         $d->{serial} = 1 if conf_has_serial($conf);
3032
3033         $res->{$vmid} = $d;
3034     }
3035
3036     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3037     foreach my $dev (keys %$netdev) {
3038         next if $dev !~ m/^tap([1-9]\d*)i/;
3039         my $vmid = $1;
3040         my $d = $res->{$vmid};
3041         next if !$d;
3042
3043         $d->{netout} += $netdev->{$dev}->{receive};
3044         $d->{netin} += $netdev->{$dev}->{transmit};
3045
3046         if ($full) {
3047             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3048             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3049         }
3050
3051     }
3052
3053     my $ctime = gettimeofday;
3054
3055     foreach my $vmid (keys %$list) {
3056
3057         my $d = $res->{$vmid};
3058         my $pid = $d->{pid};
3059         next if !$pid;
3060
3061         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3062         next if !$pstat; # not running
3063
3064         my $used = $pstat->{utime} + $pstat->{stime};
3065
3066         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3067
3068         if ($pstat->{vsize}) {
3069             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3070         }
3071
3072         my $old = $last_proc_pid_stat->{$pid};
3073         if (!$old) {
3074             $last_proc_pid_stat->{$pid} = {
3075                 time => $ctime,
3076                 used => $used,
3077                 cpu => 0,
3078             };
3079             next;
3080         }
3081
3082         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3083
3084         if ($dtime > 1000) {
3085             my $dutime = $used -  $old->{used};
3086
3087             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3088             $last_proc_pid_stat->{$pid} = {
3089                 time => $ctime,
3090                 used => $used,
3091                 cpu => $d->{cpu},
3092             };
3093         } else {
3094             $d->{cpu} = $old->{cpu};
3095         }
3096     }
3097
3098     return $res if !$full;
3099
3100     my $qmpclient = PVE::QMPClient->new();
3101
3102     my $ballooncb = sub {
3103         my ($vmid, $resp) = @_;
3104
3105         my $info = $resp->{'return'};
3106         return if !$info->{max_mem};
3107
3108         my $d = $res->{$vmid};
3109
3110         # use memory assigned to VM
3111         $d->{maxmem} = $info->{max_mem};
3112         $d->{balloon} = $info->{actual};
3113
3114         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3115             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3116             $d->{freemem} = $info->{free_mem};
3117         }
3118
3119         $d->{ballooninfo} = $info;
3120     };
3121
3122     my $blockstatscb = sub {
3123         my ($vmid, $resp) = @_;
3124         my $data = $resp->{'return'} || [];
3125         my $totalrdbytes = 0;
3126         my $totalwrbytes = 0;
3127
3128         for my $blockstat (@$data) {
3129             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3130             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3131
3132             $blockstat->{device} =~ s/drive-//;
3133             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3134         }
3135         $res->{$vmid}->{diskread} = $totalrdbytes;
3136         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3137     };
3138
3139     my $statuscb = sub {
3140         my ($vmid, $resp) = @_;
3141
3142         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3143         # this fails if ballon driver is not loaded, so this must be
3144         # the last commnand (following command are aborted if this fails).
3145         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3146
3147         my $status = 'unknown';
3148         if (!defined($status = $resp->{'return'}->{status})) {
3149             warn "unable to get VM status\n";
3150             return;
3151         }
3152
3153         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3154     };
3155
3156     foreach my $vmid (keys %$list) {
3157         next if $opt_vmid && ($vmid ne $opt_vmid);
3158         next if !$res->{$vmid}->{pid}; # not running
3159         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3160     }
3161
3162     $qmpclient->queue_execute(undef, 2);
3163
3164     foreach my $vmid (keys %$list) {
3165         next if $opt_vmid && ($vmid ne $opt_vmid);
3166         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3167     }
3168
3169     return $res;
3170 }
3171
3172 sub foreach_drive {
3173     my ($conf, $func, @param) = @_;
3174
3175     foreach my $ds (valid_drive_names()) {
3176         next if !defined($conf->{$ds});
3177
3178         my $drive = parse_drive($ds, $conf->{$ds});
3179         next if !$drive;
3180
3181         &$func($ds, $drive, @param);
3182     }
3183 }
3184
3185 sub foreach_volid {
3186     my ($conf, $func, @param) = @_;
3187
3188     my $volhash = {};
3189
3190     my $test_volid = sub {
3191         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3192
3193         return if !$volid;
3194
3195         $volhash->{$volid}->{cdrom} //= 1;
3196         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3197
3198         $volhash->{$volid}->{replicate} //= 0;
3199         $volhash->{$volid}->{replicate} = 1 if $replicate;
3200
3201         $volhash->{$volid}->{shared} //= 0;
3202         $volhash->{$volid}->{shared} = 1 if $shared;
3203
3204         $volhash->{$volid}->{referenced_in_config} //= 0;
3205         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3206
3207         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3208             if defined($snapname);
3209     };
3210
3211     foreach_drive($conf, sub {
3212         my ($ds, $drive) = @_;
3213         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3214     });
3215
3216     foreach my $snapname (keys %{$conf->{snapshots}}) {
3217         my $snap = $conf->{snapshots}->{$snapname};
3218         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3219         foreach_drive($snap, sub {
3220             my ($ds, $drive) = @_;
3221             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3222         });
3223     }
3224
3225     foreach my $volid (keys %$volhash) {
3226         &$func($volid, $volhash->{$volid}, @param);
3227     }
3228 }
3229
3230 sub conf_has_serial {
3231     my ($conf) = @_;
3232
3233     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3234         if ($conf->{"serial$i"}) {
3235             return 1;
3236         }
3237     }
3238
3239     return 0;
3240 }
3241
3242 sub vga_conf_has_spice {
3243     my ($vga) = @_;
3244
3245     my $vgaconf = parse_vga($vga);
3246     my $vgatype = $vgaconf->{type};
3247     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3248
3249     return $1 || 1;
3250 }
3251
3252 sub config_to_command {
3253     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3254
3255     my $cmd = [];
3256     my $globalFlags = [];
3257     my $machineFlags = [];
3258     my $rtcFlags = [];
3259     my $cpuFlags = [];
3260     my $devices = [];
3261     my $pciaddr = '';
3262     my $bridges = {};
3263     my $kvmver = kvm_user_version();
3264     my $vernum = 0; # unknown
3265     my $ostype = $conf->{ostype};
3266     my $winversion = windows_version($ostype);
3267     my $kvm = $conf->{kvm} // 1;
3268
3269     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3270
3271     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3272         $vernum = $1*1000000+$2*1000;
3273     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3274         $vernum = $1*1000000+$2*1000+$3;
3275     }
3276
3277     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3278
3279     my $have_ovz = -f '/proc/vz/vestat';
3280
3281     my $q35 = machine_type_is_q35($conf);
3282     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3283     my $machine_type = $forcemachine || $conf->{machine};
3284     my $use_old_bios_files = undef;
3285     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3286
3287     my $cpuunits = defined($conf->{cpuunits}) ?
3288             $conf->{cpuunits} : $defaults->{cpuunits};
3289
3290     push @$cmd, '/usr/bin/kvm';
3291
3292     push @$cmd, '-id', $vmid;
3293
3294     my $vmname = $conf->{name} || "vm$vmid";
3295
3296     push @$cmd, '-name', $vmname;
3297
3298     my $use_virtio = 0;
3299
3300     my $qmpsocket = qmp_socket($vmid);
3301     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3302     push @$cmd, '-mon', "chardev=qmp,mode=control";
3303
3304     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3305         my $eventsocket = qmp_socket($vmid, 0, 'event');
3306         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3307         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3308     }
3309
3310     push @$cmd, '-pidfile' , pidfile_name($vmid);
3311
3312     push @$cmd, '-daemonize';
3313
3314     if ($conf->{smbios1}) {
3315         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3316     }
3317
3318     if ($conf->{vmgenid}) {
3319         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3320     }
3321
3322     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3323         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3324
3325         my $path;
3326         my $format;
3327         if (my $efidisk = $conf->{efidisk0}) {
3328             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3329             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3330             $format = $d->{format};
3331             if ($storeid) {
3332                 $path = PVE::Storage::path($storecfg, $d->{file});
3333                 if (!defined($format)) {
3334                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3335                     $format = qemu_img_format($scfg, $volname);
3336                 }
3337             } else {
3338                 $path = $d->{file};
3339                 die "efidisk format must be specified\n"
3340                     if !defined($format);
3341             }
3342         } else {
3343             warn "no efidisk configured! Using temporary efivars disk.\n";
3344             $path = "/tmp/$vmid-ovmf.fd";
3345             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3346             $format = 'raw';
3347         }
3348
3349         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3350         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3351     }
3352
3353
3354     # add usb controllers
3355     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3356     push @$devices, @usbcontrollers if @usbcontrollers;
3357     my $vga = parse_vga($conf->{vga});
3358
3359     my $qxlnum = vga_conf_has_spice($conf->{vga});
3360     $vga->{type} = 'qxl' if $qxlnum;
3361
3362     if (!$vga->{type}) {
3363         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3364             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3365         } else {
3366             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3367         }
3368     }
3369
3370     # enable absolute mouse coordinates (needed by vnc)
3371     my $tablet;
3372     if (defined($conf->{tablet})) {
3373         $tablet = $conf->{tablet};
3374     } else {
3375         $tablet = $defaults->{tablet};
3376         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3377         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3378     }
3379
3380     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3381
3382     my $kvm_off = 0;
3383     my $gpu_passthrough;
3384
3385     # host pci devices
3386     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3387         my $d = parse_hostpci($conf->{"hostpci$i"});
3388         next if !$d;
3389
3390         my $pcie = $d->{pcie};
3391         if($pcie){
3392             die "q35 machine model is not enabled" if !$q35;
3393             $pciaddr = print_pcie_addr("hostpci$i");
3394         }else{
3395             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3396         }
3397
3398         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3399         my $romfile = $d->{romfile};
3400
3401         my $xvga = '';
3402         if ($d->{'x-vga'}) {
3403             $xvga = ',x-vga=on';
3404             $kvm_off = 1;
3405             $vga->{type} = 'none';
3406             $gpu_passthrough = 1;
3407
3408             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3409                 $xvga = "";
3410             }
3411         }
3412         my $pcidevices = $d->{pciid};
3413         my $multifunction = 1 if @$pcidevices > 1;
3414
3415         my $j=0;
3416         foreach my $pcidevice (@$pcidevices) {
3417
3418             my $id = "hostpci$i";
3419             $id .= ".$j" if $multifunction;
3420             my $addr = $pciaddr;
3421             $addr .= ".$j" if $multifunction;
3422             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3423
3424             if($j == 0){
3425                 $devicestr .= "$rombar$xvga";
3426                 $devicestr .= ",multifunction=on" if $multifunction;
3427                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3428             }
3429
3430             push @$devices, '-device', $devicestr;
3431             $j++;
3432         }
3433     }
3434
3435     # usb devices
3436     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3437     push @$devices, @usbdevices if @usbdevices;
3438     # serial devices
3439     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3440         if (my $path = $conf->{"serial$i"}) {
3441             if ($path eq 'socket') {
3442                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3443                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3444                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3445             } else {
3446                 die "no such serial device\n" if ! -c $path;
3447                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3448                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3449             }
3450         }
3451     }
3452
3453     # parallel devices
3454     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3455         if (my $path = $conf->{"parallel$i"}) {
3456             die "no such parallel device\n" if ! -c $path;
3457             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3458             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3459             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3460         }
3461     }
3462
3463
3464     my $sockets = 1;
3465     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3466     $sockets = $conf->{sockets} if  $conf->{sockets};
3467
3468     my $cores = $conf->{cores} || 1;
3469
3470     my $maxcpus = $sockets * $cores;
3471
3472     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3473
3474     my $allowed_vcpus = $cpuinfo->{cpus};
3475
3476     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3477         if ($allowed_vcpus < $maxcpus);
3478
3479     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3480
3481         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3482         for (my $i = 2; $i <= $vcpus; $i++)  {
3483             my $cpustr = print_cpu_device($conf,$i);
3484             push @$cmd, '-device', $cpustr;
3485         }
3486
3487     } else {
3488
3489         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3490     }
3491     push @$cmd, '-nodefaults';
3492
3493     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3494
3495     my $bootindex_hash = {};
3496     my $i = 1;
3497     foreach my $o (split(//, $bootorder)) {
3498         $bootindex_hash->{$o} = $i*100;
3499         $i++;
3500     }
3501
3502     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3503
3504     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3505
3506     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3507
3508     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga ne 'none'){
3509         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3510         my $socket = vnc_socket($vmid);
3511         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3512     } else {
3513         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3514         push @$cmd, '-nographic';
3515     }
3516
3517     # time drift fix
3518     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3519
3520     my $useLocaltime = $conf->{localtime};
3521
3522     if ($winversion >= 5) { # windows
3523         $useLocaltime = 1 if !defined($conf->{localtime});
3524
3525         # use time drift fix when acpi is enabled
3526         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3527             $tdf = 1 if !defined($conf->{tdf});
3528         }
3529     }
3530
3531     if ($winversion >= 6) {
3532         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3533         push @$cmd, '-no-hpet';
3534     }
3535
3536     push @$rtcFlags, 'driftfix=slew' if $tdf;
3537
3538     if (!$kvm) {
3539         push @$machineFlags, 'accel=tcg';
3540     }
3541
3542     if ($machine_type) {
3543         push @$machineFlags, "type=${machine_type}";
3544     }
3545
3546     if ($conf->{startdate}) {
3547         push @$rtcFlags, "base=$conf->{startdate}";
3548     } elsif ($useLocaltime) {
3549         push @$rtcFlags, 'base=localtime';
3550     }
3551
3552     my $cpu = $kvm ? "kvm64" : "qemu64";
3553     if (my $cputype = $conf->{cpu}) {
3554         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3555             or die "Cannot parse cpu description: $cputype\n";
3556         $cpu = $cpuconf->{cputype};
3557         $kvm_off = 1 if $cpuconf->{hidden};
3558
3559         if (defined(my $flags = $cpuconf->{flags})) {
3560             push @$cpuFlags, split(";", $flags);
3561         }
3562     }
3563
3564     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3565
3566     push @$cpuFlags , '-x2apic'
3567         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3568
3569     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3570
3571     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3572
3573     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3574
3575         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3576         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3577     }
3578
3579     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3580
3581     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3582
3583     push @$cpuFlags, 'kvm=off' if $kvm_off;
3584
3585     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3586         die "internal error"; # should not happen
3587
3588     push @$cpuFlags, "vendor=${cpu_vendor}"
3589         if $cpu_vendor ne 'default';
3590
3591     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3592
3593     push @$cmd, '-cpu', $cpu;
3594
3595     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3596
3597     push @$cmd, '-S' if $conf->{freeze};
3598
3599     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3600
3601     # enable sound
3602     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3603     #push @$cmd, '-soundhw', 'es1370';
3604     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3605
3606     if (parse_guest_agent($conf)->{enabled}) {
3607         my $qgasocket = qmp_socket($vmid, 1);
3608         my $pciaddr = print_pci_addr("qga0", $bridges);
3609         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3610         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3611         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3612     }
3613
3614     my $spice_port;
3615
3616     if ($qxlnum) {
3617         if ($qxlnum > 1) {
3618             if ($winversion){
3619                 for(my $i = 1; $i < $qxlnum; $i++){
3620                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3621                 }
3622             } else {
3623                 # assume other OS works like Linux
3624                 my ($ram, $vram) = ("134217728", "67108864");
3625                 if ($vga->{memory}) {
3626                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3627                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3628                 }
3629                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3630                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3631             }
3632         }
3633
3634         my $pciaddr = print_pci_addr("spice", $bridges);
3635
3636         my $nodename = PVE::INotify::nodename();
3637         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3638         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3639         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3640         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3641         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3642
3643         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3644
3645         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3646         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3647         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3648     }
3649
3650     # enable balloon by default, unless explicitly disabled
3651     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3652         $pciaddr = print_pci_addr("balloon0", $bridges);
3653         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3654     }
3655
3656     if ($conf->{watchdog}) {
3657         my $wdopts = parse_watchdog($conf->{watchdog});
3658         $pciaddr = print_pci_addr("watchdog", $bridges);
3659         my $watchdog = $wdopts->{model} || 'i6300esb';
3660         push @$devices, '-device', "$watchdog$pciaddr";
3661         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3662     }
3663
3664     my $vollist = [];
3665     my $scsicontroller = {};
3666     my $ahcicontroller = {};
3667     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3668
3669     # Add iscsi initiator name if available
3670     if (my $initiator = get_initiator_name()) {
3671         push @$devices, '-iscsi', "initiator-name=$initiator";
3672     }
3673
3674     foreach_drive($conf, sub {
3675         my ($ds, $drive) = @_;
3676
3677         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3678             push @$vollist, $drive->{file};
3679         }
3680
3681         # ignore efidisk here, already added in bios/fw handling code above
3682         return if $drive->{interface} eq 'efidisk';
3683
3684         $use_virtio = 1 if $ds =~ m/^virtio/;
3685
3686         if (drive_is_cdrom ($drive)) {
3687             if ($bootindex_hash->{d}) {
3688                 $drive->{bootindex} = $bootindex_hash->{d};
3689                 $bootindex_hash->{d} += 1;
3690             }
3691         } else {
3692             if ($bootindex_hash->{c}) {
3693                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3694                 $bootindex_hash->{c} += 1;
3695             }
3696         }
3697
3698         if($drive->{interface} eq 'virtio'){
3699            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3700         }
3701
3702         if ($drive->{interface} eq 'scsi') {
3703
3704             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3705
3706             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3707             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3708
3709             my $iothread = '';
3710             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3711                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3712                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3713             } elsif ($drive->{iothread}) {
3714                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3715             }
3716
3717             my $queues = '';
3718             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3719                 $queues = ",num_queues=$drive->{queues}";
3720             }
3721
3722             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3723             $scsicontroller->{$controller}=1;
3724         }
3725
3726         if ($drive->{interface} eq 'sata') {
3727            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3728            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3729            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3730            $ahcicontroller->{$controller}=1;
3731         }
3732
3733         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3734         push @$devices, '-drive',$drive_cmd;
3735         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3736     });
3737
3738     for (my $i = 0; $i < $MAX_NETS; $i++) {
3739          next if !$conf->{"net$i"};
3740          my $d = parse_net($conf->{"net$i"});
3741          next if !$d;
3742
3743          $use_virtio = 1 if $d->{model} eq 'virtio';
3744
3745          if ($bootindex_hash->{n}) {
3746             $d->{bootindex} = $bootindex_hash->{n};
3747             $bootindex_hash->{n} += 1;
3748          }
3749
3750          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3751          push @$devices, '-netdev', $netdevfull;
3752
3753          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3754          push @$devices, '-device', $netdevicefull;
3755     }
3756
3757     if (!$q35) {
3758         # add pci bridges
3759         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3760            $bridges->{1} = 1;
3761            $bridges->{2} = 1;
3762         }
3763
3764         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3765
3766         while (my ($k, $v) = each %$bridges) {
3767             $pciaddr = print_pci_addr("pci.$k");
3768             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3769         }
3770     }
3771
3772     # add custom args
3773     if ($conf->{args}) {
3774         my $aa = PVE::Tools::split_args($conf->{args});
3775         push @$cmd, @$aa;
3776     }
3777
3778     push @$cmd, @$devices;
3779     push @$cmd, '-rtc', join(',', @$rtcFlags)
3780         if scalar(@$rtcFlags);
3781     push @$cmd, '-machine', join(',', @$machineFlags)
3782         if scalar(@$machineFlags);
3783     push @$cmd, '-global', join(',', @$globalFlags)
3784         if scalar(@$globalFlags);
3785
3786     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3787 }
3788
3789 sub vnc_socket {
3790     my ($vmid) = @_;
3791     return "${var_run_tmpdir}/$vmid.vnc";
3792 }
3793
3794 sub spice_port {
3795     my ($vmid) = @_;
3796
3797     my $res = vm_mon_cmd($vmid, 'query-spice');
3798
3799     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3800 }
3801
3802 sub qmp_socket {
3803     my ($vmid, $qga, $name) = @_;
3804     my $sockettype = $qga ? 'qga' : 'qmp';
3805     my $ext = $name ? '-'.$name : '';
3806     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3807 }
3808
3809 sub pidfile_name {
3810     my ($vmid) = @_;
3811     return "${var_run_tmpdir}/$vmid.pid";
3812 }
3813
3814 sub vm_devices_list {
3815     my ($vmid) = @_;
3816
3817     my $res = vm_mon_cmd($vmid, 'query-pci');
3818     my $devices_to_check = [];
3819     my $devices = {};
3820     foreach my $pcibus (@$res) {
3821         push @$devices_to_check, @{$pcibus->{devices}},
3822     }
3823
3824     while (@$devices_to_check) {
3825         my $to_check = [];
3826         for my $d (@$devices_to_check) {
3827             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3828             next if !$d->{'pci_bridge'};
3829
3830             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3831             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3832         }
3833         $devices_to_check = $to_check;
3834     }
3835
3836     my $resblock = vm_mon_cmd($vmid, 'query-block');
3837     foreach my $block (@$resblock) {
3838         if($block->{device} =~ m/^drive-(\S+)/){
3839                 $devices->{$1} = 1;
3840         }
3841     }
3842
3843     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3844     foreach my $mice (@$resmice) {
3845         if ($mice->{name} eq 'QEMU HID Tablet') {
3846             $devices->{tablet} = 1;
3847             last;
3848         }
3849     }
3850
3851     # for usb devices there is no query-usb
3852     # but we can iterate over the entries in
3853     # qom-list path=/machine/peripheral
3854     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3855     foreach my $per (@$resperipheral) {
3856         if ($per->{name} =~ m/^usb\d+$/) {
3857             $devices->{$per->{name}} = 1;
3858         }
3859     }
3860
3861     return $devices;
3862 }
3863
3864 sub vm_deviceplug {
3865     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3866
3867     my $q35 = machine_type_is_q35($conf);
3868
3869     my $devices_list = vm_devices_list($vmid);
3870     return 1 if defined($devices_list->{$deviceid});
3871
3872     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3873
3874     if ($deviceid eq 'tablet') {
3875
3876         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3877
3878     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3879
3880         die "usb hotplug currently not reliable\n";
3881         # since we can't reliably hot unplug all added usb devices
3882         # and usb passthrough disables live migration
3883         # we disable usb hotplugging for now
3884         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3885
3886     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3887
3888         qemu_iothread_add($vmid, $deviceid, $device);
3889
3890         qemu_driveadd($storecfg, $vmid, $device);
3891         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3892
3893         qemu_deviceadd($vmid, $devicefull);
3894         eval { qemu_deviceaddverify($vmid, $deviceid); };
3895         if (my $err = $@) {
3896             eval { qemu_drivedel($vmid, $deviceid); };
3897             warn $@ if $@;
3898             die $err;
3899         }
3900
3901     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3902
3903
3904         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3905         my $pciaddr = print_pci_addr($deviceid);
3906         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3907
3908         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3909
3910         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3911             qemu_iothread_add($vmid, $deviceid, $device);
3912             $devicefull .= ",iothread=iothread-$deviceid";
3913         }
3914
3915         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3916             $devicefull .= ",num_queues=$device->{queues}";
3917         }
3918
3919         qemu_deviceadd($vmid, $devicefull);
3920         qemu_deviceaddverify($vmid, $deviceid);
3921
3922     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3923
3924         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3925         qemu_driveadd($storecfg, $vmid, $device);
3926
3927         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3928         eval { qemu_deviceadd($vmid, $devicefull); };
3929         if (my $err = $@) {
3930             eval { qemu_drivedel($vmid, $deviceid); };
3931             warn $@ if $@;
3932             die $err;
3933         }
3934
3935     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3936
3937         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3938
3939         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3940         my $use_old_bios_files = undef;
3941         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3942
3943         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3944         qemu_deviceadd($vmid, $netdevicefull);
3945         eval { qemu_deviceaddverify($vmid, $deviceid); };
3946         if (my $err = $@) {
3947             eval { qemu_netdevdel($vmid, $deviceid); };
3948             warn $@ if $@;
3949             die $err;
3950         }
3951
3952     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3953
3954         my $bridgeid = $2;
3955         my $pciaddr = print_pci_addr($deviceid);
3956         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3957
3958         qemu_deviceadd($vmid, $devicefull);
3959         qemu_deviceaddverify($vmid, $deviceid);
3960
3961     } else {
3962         die "can't hotplug device '$deviceid'\n";
3963     }
3964
3965     return 1;
3966 }
3967
3968 # fixme: this should raise exceptions on error!
3969 sub vm_deviceunplug {
3970     my ($vmid, $conf, $deviceid) = @_;
3971
3972     my $devices_list = vm_devices_list($vmid);
3973     return 1 if !defined($devices_list->{$deviceid});
3974
3975     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3976
3977     if ($deviceid eq 'tablet') {
3978
3979         qemu_devicedel($vmid, $deviceid);
3980
3981     } elsif ($deviceid =~ m/^usb\d+$/) {
3982
3983         die "usb hotplug currently not reliable\n";
3984         # when unplugging usb devices this way,
3985         # there may be remaining usb controllers/hubs
3986         # so we disable it for now
3987         qemu_devicedel($vmid, $deviceid);
3988         qemu_devicedelverify($vmid, $deviceid);
3989
3990     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3991
3992         qemu_devicedel($vmid, $deviceid);
3993         qemu_devicedelverify($vmid, $deviceid);
3994         qemu_drivedel($vmid, $deviceid);
3995         qemu_iothread_del($conf, $vmid, $deviceid);
3996
3997     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3998
3999         qemu_devicedel($vmid, $deviceid);
4000         qemu_devicedelverify($vmid, $deviceid);
4001         qemu_iothread_del($conf, $vmid, $deviceid);
4002
4003     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4004
4005         qemu_devicedel($vmid, $deviceid);
4006         qemu_drivedel($vmid, $deviceid);
4007         qemu_deletescsihw($conf, $vmid, $deviceid);
4008
4009     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4010
4011         qemu_devicedel($vmid, $deviceid);
4012         qemu_devicedelverify($vmid, $deviceid);
4013         qemu_netdevdel($vmid, $deviceid);
4014
4015     } else {
4016         die "can't unplug device '$deviceid'\n";
4017     }
4018
4019     return 1;
4020 }
4021
4022 sub qemu_deviceadd {
4023     my ($vmid, $devicefull) = @_;
4024
4025     $devicefull = "driver=".$devicefull;
4026     my %options =  split(/[=,]/, $devicefull);
4027
4028     vm_mon_cmd($vmid, "device_add" , %options);
4029 }
4030
4031 sub qemu_devicedel {
4032     my ($vmid, $deviceid) = @_;
4033
4034     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4035 }
4036
4037 sub qemu_iothread_add {
4038     my($vmid, $deviceid, $device) = @_;
4039
4040     if ($device->{iothread}) {
4041         my $iothreads = vm_iothreads_list($vmid);
4042         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4043     }
4044 }
4045
4046 sub qemu_iothread_del {
4047     my($conf, $vmid, $deviceid) = @_;
4048
4049     my $device = parse_drive($deviceid, $conf->{$deviceid});
4050     if ($device->{iothread}) {
4051         my $iothreads = vm_iothreads_list($vmid);
4052         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4053     }
4054 }
4055
4056 sub qemu_objectadd {
4057     my($vmid, $objectid, $qomtype) = @_;
4058
4059     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4060
4061     return 1;
4062 }
4063
4064 sub qemu_objectdel {
4065     my($vmid, $objectid) = @_;
4066
4067     vm_mon_cmd($vmid, "object-del", id => $objectid);
4068
4069     return 1;
4070 }
4071
4072 sub qemu_driveadd {
4073     my ($storecfg, $vmid, $device) = @_;
4074
4075     my $drive = print_drive_full($storecfg, $vmid, $device);
4076     $drive =~ s/\\/\\\\/g;
4077     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4078
4079     # If the command succeeds qemu prints: "OK"
4080     return 1 if $ret =~ m/OK/s;
4081
4082     die "adding drive failed: $ret\n";
4083 }
4084
4085 sub qemu_drivedel {
4086     my($vmid, $deviceid) = @_;
4087
4088     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4089     $ret =~ s/^\s+//;
4090
4091     return 1 if $ret eq "";
4092
4093     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4094     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4095
4096     die "deleting drive $deviceid failed : $ret\n";
4097 }
4098
4099 sub qemu_deviceaddverify {
4100     my ($vmid, $deviceid) = @_;
4101
4102     for (my $i = 0; $i <= 5; $i++) {
4103          my $devices_list = vm_devices_list($vmid);
4104          return 1 if defined($devices_list->{$deviceid});
4105          sleep 1;
4106     }
4107
4108     die "error on hotplug device '$deviceid'\n";
4109 }
4110
4111
4112 sub qemu_devicedelverify {
4113     my ($vmid, $deviceid) = @_;
4114
4115     # need to verify that the device is correctly removed as device_del
4116     # is async and empty return is not reliable
4117
4118     for (my $i = 0; $i <= 5; $i++) {
4119          my $devices_list = vm_devices_list($vmid);
4120          return 1 if !defined($devices_list->{$deviceid});
4121          sleep 1;
4122     }
4123
4124     die "error on hot-unplugging device '$deviceid'\n";
4125 }
4126
4127 sub qemu_findorcreatescsihw {
4128     my ($storecfg, $conf, $vmid, $device) = @_;
4129
4130     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4131
4132     my $scsihwid="$controller_prefix$controller";
4133     my $devices_list = vm_devices_list($vmid);
4134
4135     if(!defined($devices_list->{$scsihwid})) {
4136         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4137     }
4138
4139     return 1;
4140 }
4141
4142 sub qemu_deletescsihw {
4143     my ($conf, $vmid, $opt) = @_;
4144
4145     my $device = parse_drive($opt, $conf->{$opt});
4146
4147     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4148         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4149         return 1;
4150     }
4151
4152     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4153
4154     my $devices_list = vm_devices_list($vmid);
4155     foreach my $opt (keys %{$devices_list}) {
4156         if (PVE::QemuServer::is_valid_drivename($opt)) {
4157             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4158             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4159                 return 1;
4160             }
4161         }
4162     }
4163
4164     my $scsihwid="scsihw$controller";
4165
4166     vm_deviceunplug($vmid, $conf, $scsihwid);
4167
4168     return 1;
4169 }
4170
4171 sub qemu_add_pci_bridge {
4172     my ($storecfg, $conf, $vmid, $device) = @_;
4173
4174     my $bridges = {};
4175
4176     my $bridgeid;
4177
4178     print_pci_addr($device, $bridges);
4179
4180     while (my ($k, $v) = each %$bridges) {
4181         $bridgeid = $k;
4182     }
4183     return 1 if !defined($bridgeid) || $bridgeid < 1;
4184
4185     my $bridge = "pci.$bridgeid";
4186     my $devices_list = vm_devices_list($vmid);
4187
4188     if (!defined($devices_list->{$bridge})) {
4189         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4190     }
4191
4192     return 1;
4193 }
4194
4195 sub qemu_set_link_status {
4196     my ($vmid, $device, $up) = @_;
4197
4198     vm_mon_cmd($vmid, "set_link", name => $device,
4199                up => $up ? JSON::true : JSON::false);
4200 }
4201
4202 sub qemu_netdevadd {
4203     my ($vmid, $conf, $device, $deviceid) = @_;
4204
4205     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4206     my %options =  split(/[=,]/, $netdev);
4207
4208     vm_mon_cmd($vmid, "netdev_add",  %options);
4209     return 1;
4210 }
4211
4212 sub qemu_netdevdel {
4213     my ($vmid, $deviceid) = @_;
4214
4215     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4216 }
4217
4218 sub qemu_usb_hotplug {
4219     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4220
4221     return if !$device;
4222
4223     # remove the old one first
4224     vm_deviceunplug($vmid, $conf, $deviceid);
4225
4226     # check if xhci controller is necessary and available
4227     if ($device->{usb3}) {
4228
4229         my $devicelist = vm_devices_list($vmid);
4230
4231         if (!$devicelist->{xhci}) {
4232             my $pciaddr = print_pci_addr("xhci");
4233             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4234         }
4235     }
4236     my $d = parse_usb_device($device->{host});
4237     $d->{usb3} = $device->{usb3};
4238
4239     # add the new one
4240     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4241 }
4242
4243 sub qemu_cpu_hotplug {
4244     my ($vmid, $conf, $vcpus) = @_;
4245
4246     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4247
4248     my $sockets = 1;
4249     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4250     $sockets = $conf->{sockets} if  $conf->{sockets};
4251     my $cores = $conf->{cores} || 1;
4252     my $maxcpus = $sockets * $cores;
4253
4254     $vcpus = $maxcpus if !$vcpus;
4255
4256     die "you can't add more vcpus than maxcpus\n"
4257         if $vcpus > $maxcpus;
4258
4259     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4260
4261     if ($vcpus < $currentvcpus) {
4262
4263         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4264
4265             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4266                 qemu_devicedel($vmid, "cpu$i");
4267                 my $retry = 0;
4268                 my $currentrunningvcpus = undef;
4269                 while (1) {
4270                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4271                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4272                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4273                     $retry++;
4274                     sleep 1;
4275                 }
4276                 #update conf after each succesfull cpu unplug
4277                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4278                 PVE::QemuConfig->write_config($vmid, $conf);
4279             }
4280         } else {
4281             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4282         }
4283
4284         return;
4285     }
4286
4287     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4288     die "vcpus in running vm does not match its configuration\n"
4289         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4290
4291     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4292
4293         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4294             my $cpustr = print_cpu_device($conf, $i);
4295             qemu_deviceadd($vmid, $cpustr);
4296
4297             my $retry = 0;
4298             my $currentrunningvcpus = undef;
4299             while (1) {
4300                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4301                 last if scalar(@{$currentrunningvcpus}) == $i;
4302                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4303                 sleep 1;
4304                 $retry++;
4305             }
4306             #update conf after each succesfull cpu hotplug
4307             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4308             PVE::QemuConfig->write_config($vmid, $conf);
4309         }
4310     } else {
4311
4312         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4313             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4314         }
4315     }
4316 }
4317
4318 sub qemu_block_set_io_throttle {
4319     my ($vmid, $deviceid,
4320         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4321         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4322         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4323         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4324
4325     return if !check_running($vmid) ;
4326
4327     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4328         bps => int($bps),
4329         bps_rd => int($bps_rd),
4330         bps_wr => int($bps_wr),
4331         iops => int($iops),
4332         iops_rd => int($iops_rd),
4333         iops_wr => int($iops_wr),
4334         bps_max => int($bps_max),
4335         bps_rd_max => int($bps_rd_max),
4336         bps_wr_max => int($bps_wr_max),
4337         iops_max => int($iops_max),
4338         iops_rd_max => int($iops_rd_max),
4339         iops_wr_max => int($iops_wr_max),
4340         bps_max_length => int($bps_max_length),
4341         bps_rd_max_length => int($bps_rd_max_length),
4342         bps_wr_max_length => int($bps_wr_max_length),
4343         iops_max_length => int($iops_max_length),
4344         iops_rd_max_length => int($iops_rd_max_length),
4345         iops_wr_max_length => int($iops_wr_max_length),
4346     );
4347
4348 }
4349
4350 # old code, only used to shutdown old VM after update
4351 sub __read_avail {
4352     my ($fh, $timeout) = @_;
4353
4354     my $sel = new IO::Select;
4355     $sel->add($fh);
4356
4357     my $res = '';
4358     my $buf;
4359
4360     my @ready;
4361     while (scalar (@ready = $sel->can_read($timeout))) {
4362         my $count;
4363         if ($count = $fh->sysread($buf, 8192)) {
4364             if ($buf =~ /^(.*)\(qemu\) $/s) {
4365                 $res .= $1;
4366                 last;
4367             } else {
4368                 $res .= $buf;
4369             }
4370         } else {
4371             if (!defined($count)) {
4372                 die "$!\n";
4373             }
4374             last;
4375         }
4376     }
4377
4378     die "monitor read timeout\n" if !scalar(@ready);
4379
4380     return $res;
4381 }
4382
4383 sub qemu_block_resize {
4384     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4385
4386     my $running = check_running($vmid);
4387
4388     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4389
4390     return if !$running;
4391
4392     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4393
4394 }
4395
4396 sub qemu_volume_snapshot {
4397     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4398
4399     my $running = check_running($vmid);
4400
4401     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4402         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4403     } else {
4404         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4405     }
4406 }
4407
4408 sub qemu_volume_snapshot_delete {
4409     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4410
4411     my $running = check_running($vmid);
4412
4413     if($running) {
4414
4415         $running = undef;
4416         my $conf = PVE::QemuConfig->load_config($vmid);
4417         foreach_drive($conf, sub {
4418             my ($ds, $drive) = @_;
4419             $running = 1 if $drive->{file} eq $volid;
4420         });
4421     }
4422
4423     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4424         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4425     } else {
4426         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4427     }
4428 }
4429
4430 sub set_migration_caps {
4431     my ($vmid) = @_;
4432
4433     my $cap_ref = [];
4434
4435     my $enabled_cap = {
4436         "auto-converge" => 1,
4437         "xbzrle" => 1,
4438         "x-rdma-pin-all" => 0,
4439         "zero-blocks" => 0,
4440         "compress" => 0
4441     };
4442
4443     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4444
4445     for my $supported_capability (@$supported_capabilities) {
4446         push @$cap_ref, {
4447             capability => $supported_capability->{capability},
4448             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4449         };
4450     }
4451
4452     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4453 }
4454
4455 my $fast_plug_option = {
4456     'lock' => 1,
4457     'name' => 1,
4458     'onboot' => 1,
4459     'shares' => 1,
4460     'startup' => 1,
4461     'description' => 1,
4462     'protection' => 1,
4463     'vmstatestorage' => 1,
4464 };
4465
4466 # hotplug changes in [PENDING]
4467 # $selection hash can be used to only apply specified options, for
4468 # example: { cores => 1 } (only apply changed 'cores')
4469 # $errors ref is used to return error messages
4470 sub vmconfig_hotplug_pending {
4471     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4472
4473     my $defaults = load_defaults();
4474
4475     # commit values which do not have any impact on running VM first
4476     # Note: those option cannot raise errors, we we do not care about
4477     # $selection and always apply them.
4478
4479     my $add_error = sub {
4480         my ($opt, $msg) = @_;
4481         $errors->{$opt} = "hotplug problem - $msg";
4482     };
4483
4484     my $changes = 0;
4485     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4486         if ($fast_plug_option->{$opt}) {
4487             $conf->{$opt} = $conf->{pending}->{$opt};
4488             delete $conf->{pending}->{$opt};
4489             $changes = 1;
4490         }
4491     }
4492
4493     if ($changes) {
4494         PVE::QemuConfig->write_config($vmid, $conf);
4495         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4496     }
4497
4498     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4499
4500     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4501     while (my ($opt, $force) = each %$pending_delete_hash) {
4502         next if $selection && !$selection->{$opt};
4503         eval {
4504             if ($opt eq 'hotplug') {
4505                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4506             } elsif ($opt eq 'tablet') {
4507                 die "skip\n" if !$hotplug_features->{usb};
4508                 if ($defaults->{tablet}) {
4509                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4510                 } else {
4511                     vm_deviceunplug($vmid, $conf, $opt);
4512                 }
4513             } elsif ($opt =~ m/^usb\d+/) {
4514                 die "skip\n";
4515                 # since we cannot reliably hot unplug usb devices
4516                 # we are disabling it
4517                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4518                 vm_deviceunplug($vmid, $conf, $opt);
4519             } elsif ($opt eq 'vcpus') {
4520                 die "skip\n" if !$hotplug_features->{cpu};
4521                 qemu_cpu_hotplug($vmid, $conf, undef);
4522             } elsif ($opt eq 'balloon') {
4523                 # enable balloon device is not hotpluggable
4524                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4525                 # here we reset the ballooning value to memory
4526                 my $balloon = $conf->{memory} || $defaults->{memory};
4527                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4528             } elsif ($fast_plug_option->{$opt}) {
4529                 # do nothing
4530             } elsif ($opt =~ m/^net(\d+)$/) {
4531                 die "skip\n" if !$hotplug_features->{network};
4532                 vm_deviceunplug($vmid, $conf, $opt);
4533             } elsif (is_valid_drivename($opt)) {
4534                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4535                 vm_deviceunplug($vmid, $conf, $opt);
4536                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4537             } elsif ($opt =~ m/^memory$/) {
4538                 die "skip\n" if !$hotplug_features->{memory};
4539                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4540             } elsif ($opt eq 'cpuunits') {
4541                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4542             } elsif ($opt eq 'cpulimit') {
4543                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4544             } else {
4545                 die "skip\n";
4546             }
4547         };
4548         if (my $err = $@) {
4549             &$add_error($opt, $err) if $err ne "skip\n";
4550         } else {
4551             # save new config if hotplug was successful
4552             delete $conf->{$opt};
4553             vmconfig_undelete_pending_option($conf, $opt);
4554             PVE::QemuConfig->write_config($vmid, $conf);
4555             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4556         }
4557     }
4558
4559     my $apply_pending_cloudinit;
4560     $apply_pending_cloudinit = sub {
4561         my ($key, $value) = @_;
4562         $apply_pending_cloudinit = sub {}; # once is enough
4563
4564         my @cloudinit_opts = keys %$confdesc_cloudinit;
4565         foreach my $opt (keys %{$conf->{pending}}) {
4566             next if !grep { $_ eq $opt } @cloudinit_opts;
4567             $conf->{$opt} = delete $conf->{pending}->{$opt};
4568         }
4569
4570         my $new_conf = { %$conf };
4571         $new_conf->{$key} = $value;
4572         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4573     };
4574
4575     foreach my $opt (keys %{$conf->{pending}}) {
4576         next if $selection && !$selection->{$opt};
4577         my $value = $conf->{pending}->{$opt};
4578         eval {
4579             if ($opt eq 'hotplug') {
4580                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4581             } elsif ($opt eq 'tablet') {
4582                 die "skip\n" if !$hotplug_features->{usb};
4583                 if ($value == 1) {
4584                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4585                 } elsif ($value == 0) {
4586                     vm_deviceunplug($vmid, $conf, $opt);
4587                 }
4588             } elsif ($opt =~ m/^usb\d+$/) {
4589                 die "skip\n";
4590                 # since we cannot reliably hot unplug usb devices
4591                 # we are disabling it
4592                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4593                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4594                 die "skip\n" if !$d;
4595                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4596             } elsif ($opt eq 'vcpus') {
4597                 die "skip\n" if !$hotplug_features->{cpu};
4598                 qemu_cpu_hotplug($vmid, $conf, $value);
4599             } elsif ($opt eq 'balloon') {
4600                 # enable/disable balloning device is not hotpluggable
4601                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4602                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4603                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4604
4605                 # allow manual ballooning if shares is set to zero
4606                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4607                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4608                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4609                 }
4610             } elsif ($opt =~ m/^net(\d+)$/) {
4611                 # some changes can be done without hotplug
4612                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4613                                     $vmid, $opt, $value);
4614             } elsif (is_valid_drivename($opt)) {
4615                 # some changes can be done without hotplug
4616                 my $drive = parse_drive($opt, $value);
4617                 if (drive_is_cloudinit($drive)) {
4618                     &$apply_pending_cloudinit($opt, $value);
4619                 }
4620                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4621                                      $vmid, $opt, $value, 1);
4622             } elsif ($opt =~ m/^memory$/) { #dimms
4623                 die "skip\n" if !$hotplug_features->{memory};
4624                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4625             } elsif ($opt eq 'cpuunits') {
4626                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4627             } elsif ($opt eq 'cpulimit') {
4628                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4629                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4630             } else {
4631                 die "skip\n";  # skip non-hot-pluggable options
4632             }
4633         };
4634         if (my $err = $@) {
4635             &$add_error($opt, $err) if $err ne "skip\n";
4636         } else {
4637             # save new config if hotplug was successful
4638             $conf->{$opt} = $value;
4639             delete $conf->{pending}->{$opt};
4640             PVE::QemuConfig->write_config($vmid, $conf);
4641             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4642         }
4643     }
4644 }
4645
4646 sub try_deallocate_drive {
4647     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4648
4649     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4650         my $volid = $drive->{file};
4651         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4652             my $sid = PVE::Storage::parse_volume_id($volid);
4653             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4654
4655             # check if the disk is really unused
4656             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4657                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4658             PVE::Storage::vdisk_free($storecfg, $volid);
4659             return 1;
4660         } else {
4661             # If vm is not owner of this disk remove from config
4662             return 1;
4663         }
4664     }
4665
4666     return undef;
4667 }
4668
4669 sub vmconfig_delete_or_detach_drive {
4670     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4671
4672     my $drive = parse_drive($opt, $conf->{$opt});
4673
4674     my $rpcenv = PVE::RPCEnvironment::get();
4675     my $authuser = $rpcenv->get_user();
4676
4677     if ($force) {
4678         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4679         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4680     } else {
4681         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4682     }
4683 }
4684
4685 sub vmconfig_apply_pending {
4686     my ($vmid, $conf, $storecfg) = @_;
4687
4688     # cold plug
4689
4690     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4691     while (my ($opt, $force) = each %$pending_delete_hash) {
4692         die "internal error" if $opt =~ m/^unused/;
4693         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4694         if (!defined($conf->{$opt})) {
4695             vmconfig_undelete_pending_option($conf, $opt);
4696             PVE::QemuConfig->write_config($vmid, $conf);
4697         } elsif (is_valid_drivename($opt)) {
4698             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4699             vmconfig_undelete_pending_option($conf, $opt);
4700             delete $conf->{$opt};
4701             PVE::QemuConfig->write_config($vmid, $conf);
4702         } else {
4703             vmconfig_undelete_pending_option($conf, $opt);
4704             delete $conf->{$opt};
4705             PVE::QemuConfig->write_config($vmid, $conf);
4706         }
4707     }
4708
4709     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4710
4711     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4712         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4713
4714         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4715             # skip if nothing changed
4716         } elsif (is_valid_drivename($opt)) {
4717             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4718                 if defined($conf->{$opt});
4719             $conf->{$opt} = $conf->{pending}->{$opt};
4720         } else {
4721             $conf->{$opt} = $conf->{pending}->{$opt};
4722         }
4723
4724         delete $conf->{pending}->{$opt};
4725         PVE::QemuConfig->write_config($vmid, $conf);
4726     }
4727 }
4728
4729 my $safe_num_ne = sub {
4730     my ($a, $b) = @_;
4731
4732     return 0 if !defined($a) && !defined($b);
4733     return 1 if !defined($a);
4734     return 1 if !defined($b);
4735
4736     return $a != $b;
4737 };
4738
4739 my $safe_string_ne = sub {
4740     my ($a, $b) = @_;
4741
4742     return 0 if !defined($a) && !defined($b);
4743     return 1 if !defined($a);
4744     return 1 if !defined($b);
4745
4746     return $a ne $b;
4747 };
4748
4749 sub vmconfig_update_net {
4750     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4751
4752     my $newnet = parse_net($value);
4753
4754     if ($conf->{$opt}) {
4755         my $oldnet = parse_net($conf->{$opt});
4756
4757         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4758             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4759             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4760             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4761
4762             # for non online change, we try to hot-unplug
4763             die "skip\n" if !$hotplug;
4764             vm_deviceunplug($vmid, $conf, $opt);
4765         } else {
4766
4767             die "internal error" if $opt !~ m/net(\d+)/;
4768             my $iface = "tap${vmid}i$1";
4769
4770             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4771                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4772                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4773                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4774                 PVE::Network::tap_unplug($iface);
4775                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4776             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4777                 # Rate can be applied on its own but any change above needs to
4778                 # include the rate in tap_plug since OVS resets everything.
4779                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4780             }
4781
4782             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4783                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4784             }
4785
4786             return 1;
4787         }
4788     }
4789
4790     if ($hotplug) {
4791         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4792     } else {
4793         die "skip\n";
4794     }
4795 }
4796
4797 sub vmconfig_update_disk {
4798     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4799
4800     # fixme: do we need force?
4801
4802     my $drive = parse_drive($opt, $value);
4803
4804     if ($conf->{$opt}) {
4805
4806         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4807
4808             my $media = $drive->{media} || 'disk';
4809             my $oldmedia = $old_drive->{media} || 'disk';
4810             die "unable to change media type\n" if $media ne $oldmedia;
4811
4812             if (!drive_is_cdrom($old_drive)) {
4813
4814                 if ($drive->{file} ne $old_drive->{file}) {
4815
4816                     die "skip\n" if !$hotplug;
4817
4818                     # unplug and register as unused
4819                     vm_deviceunplug($vmid, $conf, $opt);
4820                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4821
4822                 } else {
4823                     # update existing disk
4824
4825                     # skip non hotpluggable value
4826                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4827                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4828                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4829                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4830                         die "skip\n";
4831                     }
4832
4833                     # apply throttle
4834                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4835                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4836                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4837                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4838                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4839                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4840                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4841                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4842                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4843                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4844                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4845                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4846                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4847                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4848                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4849                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4850                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4851                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4852
4853                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4854                                                    ($drive->{mbps} || 0)*1024*1024,
4855                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4856                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4857                                                    $drive->{iops} || 0,
4858                                                    $drive->{iops_rd} || 0,
4859                                                    $drive->{iops_wr} || 0,
4860                                                    ($drive->{mbps_max} || 0)*1024*1024,
4861                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4862                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4863                                                    $drive->{iops_max} || 0,
4864                                                    $drive->{iops_rd_max} || 0,
4865                                                    $drive->{iops_wr_max} || 0,
4866                                                    $drive->{bps_max_length} || 1,
4867                                                    $drive->{bps_rd_max_length} || 1,
4868                                                    $drive->{bps_wr_max_length} || 1,
4869                                                    $drive->{iops_max_length} || 1,
4870                                                    $drive->{iops_rd_max_length} || 1,
4871                                                    $drive->{iops_wr_max_length} || 1);
4872
4873                     }
4874
4875                     return 1;
4876                 }
4877
4878             } else { # cdrom
4879
4880                 if ($drive->{file} eq 'none') {
4881                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4882                     if (drive_is_cloudinit($old_drive)) {
4883                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4884                     }
4885                 } else {
4886                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4887                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4888                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4889                 }
4890
4891                 return 1;
4892             }
4893         }
4894     }
4895
4896     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4897     # hotplug new disks
4898     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4899     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4900 }
4901
4902 sub vm_start {
4903     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4904         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4905
4906     PVE::QemuConfig->lock_config($vmid, sub {
4907         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4908
4909         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4910
4911         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4912
4913         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4914
4915         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4916             vmconfig_apply_pending($vmid, $conf, $storecfg);
4917             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4918         }
4919
4920         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4921
4922         my $defaults = load_defaults();
4923
4924         # set environment variable useful inside network script
4925         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4926
4927         my $local_volumes = {};
4928
4929         if ($targetstorage) {
4930             foreach_drive($conf, sub {
4931                 my ($ds, $drive) = @_;
4932
4933                 return if drive_is_cdrom($drive);
4934
4935                 my $volid = $drive->{file};
4936
4937                 return if !$volid;
4938
4939                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4940
4941                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4942                 return if $scfg->{shared};
4943                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4944             });
4945
4946             my $format = undef;
4947
4948             foreach my $opt (sort keys %$local_volumes) {
4949
4950                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4951                 my $drive = parse_drive($opt, $conf->{$opt});
4952
4953                 #if remote storage is specified, use default format
4954                 if ($targetstorage && $targetstorage ne "1") {
4955                     $storeid = $targetstorage;
4956                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4957                     $format = $defFormat;
4958                 } else {
4959                     #else we use same format than original
4960                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4961                     $format = qemu_img_format($scfg, $volid);
4962                 }
4963
4964                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4965                 my $newdrive = $drive;
4966                 $newdrive->{format} = $format;
4967                 $newdrive->{file} = $newvolid;
4968                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4969                 $local_volumes->{$opt} = $drivestr;
4970                 #pass drive to conf for command line
4971                 $conf->{$opt} = $drivestr;
4972             }
4973         }
4974
4975         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4976
4977         my $migrate_port = 0;
4978         my $migrate_uri;
4979         if ($statefile) {
4980             if ($statefile eq 'tcp') {
4981                 my $localip = "localhost";
4982                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4983                 my $nodename = PVE::INotify::nodename();
4984
4985                 if (!defined($migration_type)) {
4986                     if (defined($datacenterconf->{migration}->{type})) {
4987                         $migration_type = $datacenterconf->{migration}->{type};
4988                     } else {
4989                         $migration_type = 'secure';
4990                     }
4991                 }
4992
4993                 if ($migration_type eq 'insecure') {
4994                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4995                     if ($migrate_network_addr) {
4996                         $localip = $migrate_network_addr;
4997                     } else {
4998                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4999                     }
5000
5001                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5002                 }
5003
5004                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5005                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5006                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5007                 push @$cmd, '-incoming', $migrate_uri;
5008                 push @$cmd, '-S';
5009
5010             } elsif ($statefile eq 'unix') {
5011                 # should be default for secure migrations as a ssh TCP forward
5012                 # tunnel is not deterministic reliable ready and fails regurarly
5013                 # to set up in time, so use UNIX socket forwards
5014                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5015                 unlink $socket_addr;
5016
5017                 $migrate_uri = "unix:$socket_addr";
5018
5019                 push @$cmd, '-incoming', $migrate_uri;
5020                 push @$cmd, '-S';
5021
5022             } else {
5023                 push @$cmd, '-loadstate', $statefile;
5024             }
5025         } elsif ($paused) {
5026             push @$cmd, '-S';
5027         }
5028
5029         # host pci devices
5030         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5031           my $d = parse_hostpci($conf->{"hostpci$i"});
5032           next if !$d;
5033           my $pcidevices = $d->{pciid};
5034           foreach my $pcidevice (@$pcidevices) {
5035                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5036
5037                 my $info = pci_device_info("0000:$pciid");
5038                 die "IOMMU not present\n" if !check_iommu_support();
5039                 die "no pci device info for device '$pciid'\n" if !$info;
5040                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5041                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5042           }
5043         }
5044
5045         PVE::Storage::activate_volumes($storecfg, $vollist);
5046
5047         if (!check_running($vmid, 1)) {
5048             eval {
5049                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5050                     outfunc => sub {}, errfunc => sub {});
5051             };
5052         }
5053
5054         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5055                                                   : $defaults->{cpuunits};
5056
5057         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5058         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5059
5060         my %properties = (
5061             Slice => 'qemu.slice',
5062             KillMode => 'none',
5063             CPUShares => $cpuunits
5064         );
5065
5066         if (my $cpulimit = $conf->{cpulimit}) {
5067             $properties{CPUQuota} = int($cpulimit * 100);
5068         }
5069         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5070
5071         my $run_qemu = sub {
5072             PVE::Tools::run_fork sub {
5073                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5074                 run_command($cmd, %run_params);
5075             };
5076         };
5077
5078         if ($conf->{hugepages}) {
5079
5080             my $code = sub {
5081                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5082                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5083
5084                 PVE::QemuServer::Memory::hugepages_mount();
5085                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5086
5087                 eval { $run_qemu->() };
5088                 if (my $err = $@) {
5089                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5090                     die $err;
5091                 }
5092
5093                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5094             };
5095             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5096
5097         } else {
5098             eval { $run_qemu->() };
5099         }
5100
5101         if (my $err = $@) {
5102             # deactivate volumes if start fails
5103             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5104             die "start failed: $err";
5105         }
5106
5107         print "migration listens on $migrate_uri\n" if $migrate_uri;
5108
5109         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5110             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5111             warn $@ if $@;
5112         }
5113
5114         #start nbd server for storage migration
5115         if ($targetstorage) {
5116             my $nodename = PVE::INotify::nodename();
5117             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5118             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5119             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5120             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5121
5122             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5123
5124             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5125
5126             foreach my $opt (sort keys %$local_volumes) {
5127                 my $volid = $local_volumes->{$opt};
5128                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5129                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5130                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5131             }
5132         }
5133
5134         if ($migratedfrom) {
5135             eval {
5136                 set_migration_caps($vmid);
5137             };
5138             warn $@ if $@;
5139
5140             if ($spice_port) {
5141                 print "spice listens on port $spice_port\n";
5142                 if ($spice_ticket) {
5143                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5144                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5145                 }
5146             }
5147
5148         } else {
5149             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5150                 if !$statefile && $conf->{balloon};
5151
5152             foreach my $opt (keys %$conf) {
5153                 next if $opt !~  m/^net\d+$/;
5154                 my $nicconf = parse_net($conf->{$opt});
5155                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5156             }
5157         }
5158
5159         vm_mon_cmd_nocheck($vmid, 'qom-set',
5160                     path => "machine/peripheral/balloon0",
5161                     property => "guest-stats-polling-interval",
5162                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5163
5164     });
5165 }
5166
5167 sub vm_mon_cmd {
5168     my ($vmid, $execute, %params) = @_;
5169
5170     my $cmd = { execute => $execute, arguments => \%params };
5171     vm_qmp_command($vmid, $cmd);
5172 }
5173
5174 sub vm_mon_cmd_nocheck {
5175     my ($vmid, $execute, %params) = @_;
5176
5177     my $cmd = { execute => $execute, arguments => \%params };
5178     vm_qmp_command($vmid, $cmd, 1);
5179 }
5180
5181 sub vm_qmp_command {
5182     my ($vmid, $cmd, $nocheck) = @_;
5183
5184     my $res;
5185
5186     my $timeout;
5187     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5188         $timeout = $cmd->{arguments}->{timeout};
5189         delete $cmd->{arguments}->{timeout};
5190     }
5191
5192     eval {
5193         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5194         my $sname = qmp_socket($vmid);
5195         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5196             my $qmpclient = PVE::QMPClient->new();
5197
5198             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5199         } else {
5200             die "unable to open monitor socket\n";
5201         }
5202     };
5203     if (my $err = $@) {
5204         syslog("err", "VM $vmid qmp command failed - $err");
5205         die $err;
5206     }
5207
5208     return $res;
5209 }
5210
5211 sub vm_human_monitor_command {
5212     my ($vmid, $cmdline) = @_;
5213
5214     my $res;
5215
5216     my $cmd = {
5217         execute => 'human-monitor-command',
5218         arguments => { 'command-line' => $cmdline},
5219     };
5220
5221     return vm_qmp_command($vmid, $cmd);
5222 }
5223
5224 sub vm_commandline {
5225     my ($storecfg, $vmid) = @_;
5226
5227     my $conf = PVE::QemuConfig->load_config($vmid);
5228
5229     my $defaults = load_defaults();
5230
5231     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5232
5233     return PVE::Tools::cmd2string($cmd);
5234 }
5235
5236 sub vm_reset {
5237     my ($vmid, $skiplock) = @_;
5238
5239     PVE::QemuConfig->lock_config($vmid, sub {
5240
5241         my $conf = PVE::QemuConfig->load_config($vmid);
5242
5243         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5244
5245         vm_mon_cmd($vmid, "system_reset");
5246     });
5247 }
5248
5249 sub get_vm_volumes {
5250     my ($conf) = @_;
5251
5252     my $vollist = [];
5253     foreach_volid($conf, sub {
5254         my ($volid, $attr) = @_;
5255
5256         return if $volid =~ m|^/|;
5257
5258         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5259         return if !$sid;
5260
5261         push @$vollist, $volid;
5262     });
5263
5264     return $vollist;
5265 }
5266
5267 sub vm_stop_cleanup {
5268     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5269
5270     eval {
5271
5272         if (!$keepActive) {
5273             my $vollist = get_vm_volumes($conf);
5274             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5275         }
5276
5277         foreach my $ext (qw(mon qmp pid vnc qga)) {
5278             unlink "/var/run/qemu-server/${vmid}.$ext";
5279         }
5280
5281         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5282     };
5283     warn $@ if $@; # avoid errors - just warn
5284 }
5285
5286 # Note: use $nockeck to skip tests if VM configuration file exists.
5287 # We need that when migration VMs to other nodes (files already moved)
5288 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5289 sub vm_stop {
5290     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5291
5292     $force = 1 if !defined($force) && !$shutdown;
5293
5294     if ($migratedfrom){
5295         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5296         kill 15, $pid if $pid;
5297         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5298         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5299         return;
5300     }
5301
5302     PVE::QemuConfig->lock_config($vmid, sub {
5303
5304         my $pid = check_running($vmid, $nocheck);
5305         return if !$pid;
5306
5307         my $conf;
5308         if (!$nocheck) {
5309             $conf = PVE::QemuConfig->load_config($vmid);
5310             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5311             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5312                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5313                 $timeout = $opts->{down} if $opts->{down};
5314             }
5315         }
5316
5317         $timeout = 60 if !defined($timeout);
5318
5319         eval {
5320             if ($shutdown) {
5321                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5322                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5323                 } else {
5324                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5325                 }
5326             } else {
5327                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5328             }
5329         };
5330         my $err = $@;
5331
5332         if (!$err) {
5333             my $count = 0;
5334             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5335                 $count++;
5336                 sleep 1;
5337             }
5338
5339             if ($count >= $timeout) {
5340                 if ($force) {
5341                     warn "VM still running - terminating now with SIGTERM\n";
5342                     kill 15, $pid;
5343                 } else {
5344                     die "VM quit/powerdown failed - got timeout\n";
5345                 }
5346             } else {
5347                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5348                 return;
5349             }
5350         } else {
5351             if ($force) {
5352                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5353                 kill 15, $pid;
5354             } else {
5355                 die "VM quit/powerdown failed\n";
5356             }
5357         }
5358
5359         # wait again
5360         $timeout = 10;
5361
5362         my $count = 0;
5363         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5364             $count++;
5365             sleep 1;
5366         }
5367
5368         if ($count >= $timeout) {
5369             warn "VM still running - terminating now with SIGKILL\n";
5370             kill 9, $pid;
5371             sleep 1;
5372         }
5373
5374         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5375    });
5376 }
5377
5378 sub vm_suspend {
5379     my ($vmid, $skiplock) = @_;
5380
5381     PVE::QemuConfig->lock_config($vmid, sub {
5382
5383         my $conf = PVE::QemuConfig->load_config($vmid);
5384
5385         PVE::QemuConfig->check_lock($conf)
5386             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5387
5388         vm_mon_cmd($vmid, "stop");
5389     });
5390 }
5391
5392 sub vm_resume {
5393     my ($vmid, $skiplock, $nocheck) = @_;
5394
5395     PVE::QemuConfig->lock_config($vmid, sub {
5396
5397         my $res = vm_mon_cmd($vmid, 'query-status');
5398         my $resume_cmd = 'cont';
5399
5400         if ($res->{status} && $res->{status} eq 'suspended') {
5401             $resume_cmd = 'system_wakeup';
5402         }
5403
5404         if (!$nocheck) {
5405
5406             my $conf = PVE::QemuConfig->load_config($vmid);
5407
5408             PVE::QemuConfig->check_lock($conf)
5409                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5410
5411             vm_mon_cmd($vmid, $resume_cmd);
5412
5413         } else {
5414             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5415         }
5416     });
5417 }
5418
5419 sub vm_sendkey {
5420     my ($vmid, $skiplock, $key) = @_;
5421
5422     PVE::QemuConfig->lock_config($vmid, sub {
5423
5424         my $conf = PVE::QemuConfig->load_config($vmid);
5425
5426         # there is no qmp command, so we use the human monitor command
5427         vm_human_monitor_command($vmid, "sendkey $key");
5428     });
5429 }
5430
5431 sub vm_destroy {
5432     my ($storecfg, $vmid, $skiplock) = @_;
5433
5434     PVE::QemuConfig->lock_config($vmid, sub {
5435
5436         my $conf = PVE::QemuConfig->load_config($vmid);
5437
5438         if (!check_running($vmid)) {
5439             destroy_vm($storecfg, $vmid, undef, $skiplock);
5440         } else {
5441             die "VM $vmid is running - destroy failed\n";
5442         }
5443     });
5444 }
5445
5446 # pci helpers
5447
5448 sub file_write {
5449     my ($filename, $buf) = @_;
5450
5451     my $fh = IO::File->new($filename, "w");
5452     return undef if !$fh;
5453
5454     my $res = print $fh $buf;
5455
5456     $fh->close();
5457
5458     return $res;
5459 }
5460
5461 sub pci_device_info {
5462     my ($name) = @_;
5463
5464     my $res;
5465
5466     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5467     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5468
5469     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5470     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5471
5472     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5473     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5474
5475     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5476     return undef if !defined($product) || $product !~ s/^0x//;
5477
5478     $res = {
5479         name => $name,
5480         vendor => $vendor,
5481         product => $product,
5482         domain => $domain,
5483         bus => $bus,
5484         slot => $slot,
5485         func => $func,
5486         irq => $irq,
5487         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5488     };
5489
5490     return $res;
5491 }
5492
5493 sub pci_dev_reset {
5494     my ($dev) = @_;
5495
5496     my $name = $dev->{name};
5497
5498     my $fn = "$pcisysfs/devices/$name/reset";
5499
5500     return file_write($fn, "1");
5501 }
5502
5503 sub pci_dev_bind_to_vfio {
5504     my ($dev) = @_;
5505
5506     my $name = $dev->{name};
5507
5508     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5509
5510     if (!-d $vfio_basedir) {
5511         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5512     }
5513     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5514
5515     my $testdir = "$vfio_basedir/$name";
5516     return 1 if -d $testdir;
5517
5518     my $data = "$dev->{vendor} $dev->{product}";
5519     return undef if !file_write("$vfio_basedir/new_id", $data);
5520
5521     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5522     if (!file_write($fn, $name)) {
5523         return undef if -f $fn;
5524     }
5525
5526     $fn = "$vfio_basedir/bind";
5527     if (! -d $testdir) {
5528         return undef if !file_write($fn, $name);
5529     }
5530
5531     return -d $testdir;
5532 }
5533
5534 sub pci_dev_group_bind_to_vfio {
5535     my ($pciid) = @_;
5536
5537     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5538
5539     if (!-d $vfio_basedir) {
5540         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5541     }
5542     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5543
5544     # get IOMMU group devices
5545     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5546       my @devs = grep /^0000:/, readdir($D);
5547     closedir($D);
5548
5549     foreach my $pciid (@devs) {
5550         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5551
5552         # pci bridges, switches or root ports are not supported
5553         # they have a pci_bus subdirectory so skip them
5554         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5555
5556         my $info = pci_device_info($1);
5557         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5558     }
5559
5560     return 1;
5561 }
5562
5563 # vzdump restore implementaion
5564
5565 sub tar_archive_read_firstfile {
5566     my $archive = shift;
5567
5568     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5569
5570     # try to detect archive type first
5571     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5572         die "unable to open file '$archive'\n";
5573     my $firstfile = <$fh>;
5574     kill 15, $pid;
5575     close $fh;
5576
5577     die "ERROR: archive contaions no data\n" if !$firstfile;
5578     chomp $firstfile;
5579
5580     return $firstfile;
5581 }
5582
5583 sub tar_restore_cleanup {
5584     my ($storecfg, $statfile) = @_;
5585
5586     print STDERR "starting cleanup\n";
5587
5588     if (my $fd = IO::File->new($statfile, "r")) {
5589         while (defined(my $line = <$fd>)) {
5590             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5591                 my $volid = $2;
5592                 eval {
5593                     if ($volid =~ m|^/|) {
5594                         unlink $volid || die 'unlink failed\n';
5595                     } else {
5596                         PVE::Storage::vdisk_free($storecfg, $volid);
5597                     }
5598                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5599                 };
5600                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5601             } else {
5602                 print STDERR "unable to parse line in statfile - $line";
5603             }
5604         }
5605         $fd->close();
5606     }
5607 }
5608
5609 sub restore_archive {
5610     my ($archive, $vmid, $user, $opts) = @_;
5611
5612     my $format = $opts->{format};
5613     my $comp;
5614
5615     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5616         $format = 'tar' if !$format;
5617         $comp = 'gzip';
5618     } elsif ($archive =~ m/\.tar$/) {
5619         $format = 'tar' if !$format;
5620     } elsif ($archive =~ m/.tar.lzo$/) {
5621         $format = 'tar' if !$format;
5622         $comp = 'lzop';
5623     } elsif ($archive =~ m/\.vma$/) {
5624         $format = 'vma' if !$format;
5625     } elsif ($archive =~ m/\.vma\.gz$/) {
5626         $format = 'vma' if !$format;
5627         $comp = 'gzip';
5628     } elsif ($archive =~ m/\.vma\.lzo$/) {
5629         $format = 'vma' if !$format;
5630         $comp = 'lzop';
5631     } else {
5632         $format = 'vma' if !$format; # default
5633     }
5634
5635     # try to detect archive format
5636     if ($format eq 'tar') {
5637         return restore_tar_archive($archive, $vmid, $user, $opts);
5638     } else {
5639         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5640     }
5641 }
5642
5643 sub restore_update_config_line {
5644     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5645
5646     return if $line =~ m/^\#qmdump\#/;
5647     return if $line =~ m/^\#vzdump\#/;
5648     return if $line =~ m/^lock:/;
5649     return if $line =~ m/^unused\d+:/;
5650     return if $line =~ m/^parent:/;
5651     return if $line =~ m/^template:/; # restored VM is never a template
5652
5653     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5654     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5655         # try to convert old 1.X settings
5656         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5657         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5658             my ($model, $macaddr) = split(/\=/, $devconfig);
5659             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5660             my $net = {
5661                 model => $model,
5662                 bridge => "vmbr$ind",
5663                 macaddr => $macaddr,
5664             };
5665             my $netstr = print_net($net);
5666
5667             print $outfd "net$cookie->{netcount}: $netstr\n";
5668             $cookie->{netcount}++;
5669         }
5670     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5671         my ($id, $netstr) = ($1, $2);
5672         my $net = parse_net($netstr);
5673         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5674         $netstr = print_net($net);
5675         print $outfd "$id: $netstr\n";
5676     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5677         my $virtdev = $1;
5678         my $value = $3;
5679         my $di = parse_drive($virtdev, $value);
5680         if (defined($di->{backup}) && !$di->{backup}) {
5681             print $outfd "#$line";
5682         } elsif ($map->{$virtdev}) {
5683             delete $di->{format}; # format can change on restore
5684             $di->{file} = $map->{$virtdev};
5685             $value = print_drive($vmid, $di);
5686             print $outfd "$virtdev: $value\n";
5687         } else {
5688             print $outfd $line;
5689         }
5690     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5691         my $vmgenid = $1;
5692         if ($vmgenid ne '0') {
5693             # always generate a new vmgenid if there was a valid one setup
5694             $vmgenid = generate_uuid();
5695         }
5696         print $outfd "vmgenid: $vmgenid\n";
5697     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5698         my ($uuid, $uuid_str);
5699         UUID::generate($uuid);
5700         UUID::unparse($uuid, $uuid_str);
5701         my $smbios1 = parse_smbios1($2);
5702         $smbios1->{uuid} = $uuid_str;
5703         print $outfd $1.print_smbios1($smbios1)."\n";
5704     } else {
5705         print $outfd $line;
5706     }
5707 }
5708
5709 sub scan_volids {
5710     my ($cfg, $vmid) = @_;
5711
5712     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5713
5714     my $volid_hash = {};
5715     foreach my $storeid (keys %$info) {
5716         foreach my $item (@{$info->{$storeid}}) {
5717             next if !($item->{volid} && $item->{size});
5718             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5719             $volid_hash->{$item->{volid}} = $item;
5720         }
5721     }
5722
5723     return $volid_hash;
5724 }
5725
5726 sub is_volume_in_use {
5727     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5728
5729     my $path = PVE::Storage::path($storecfg, $volid);
5730
5731     my $scan_config = sub {
5732         my ($cref, $snapname) = @_;
5733
5734         foreach my $key (keys %$cref) {
5735             my $value = $cref->{$key};
5736             if (is_valid_drivename($key)) {
5737                 next if $skip_drive && $key eq $skip_drive;
5738                 my $drive = parse_drive($key, $value);
5739                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5740                 return 1 if $volid eq $drive->{file};
5741                 if ($drive->{file} =~ m!^/!) {
5742                     return 1 if $drive->{file} eq $path;
5743                 } else {
5744                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5745                     next if !$storeid;
5746                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5747                     next if !$scfg;
5748                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5749                 }
5750             }
5751         }
5752
5753         return 0;
5754     };
5755
5756     return 1 if &$scan_config($conf);
5757
5758     undef $skip_drive;
5759
5760     foreach my $snapname (keys %{$conf->{snapshots}}) {
5761         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5762     }
5763
5764     return 0;
5765 }
5766
5767 sub update_disksize {
5768     my ($vmid, $conf, $volid_hash) = @_;
5769
5770     my $changes;
5771     my $prefix = "VM $vmid:";
5772
5773     # used and unused disks
5774     my $referenced = {};
5775
5776     # Note: it is allowed to define multiple storages with same path (alias), so
5777     # we need to check both 'volid' and real 'path' (two different volid can point
5778     # to the same path).
5779
5780     my $referencedpath = {};
5781
5782     # update size info
5783     foreach my $opt (keys %$conf) {
5784         if (is_valid_drivename($opt)) {
5785             my $drive = parse_drive($opt, $conf->{$opt});
5786             my $volid = $drive->{file};
5787             next if !$volid;
5788
5789             $referenced->{$volid} = 1;
5790             if ($volid_hash->{$volid} &&
5791                 (my $path = $volid_hash->{$volid}->{path})) {
5792                 $referencedpath->{$path} = 1;
5793             }
5794
5795             next if drive_is_cdrom($drive);
5796             next if !$volid_hash->{$volid};
5797
5798             $drive->{size} = $volid_hash->{$volid}->{size};
5799             my $new = print_drive($vmid, $drive);
5800             if ($new ne $conf->{$opt}) {
5801                 $changes = 1;
5802                 $conf->{$opt} = $new;
5803                 print "$prefix update disk '$opt' information.\n";
5804             }
5805         }
5806     }
5807
5808     # remove 'unusedX' entry if volume is used
5809     foreach my $opt (keys %$conf) {
5810         next if $opt !~ m/^unused\d+$/;
5811         my $volid = $conf->{$opt};
5812         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5813         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5814             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5815             $changes = 1;
5816             delete $conf->{$opt};
5817         }
5818
5819         $referenced->{$volid} = 1;
5820         $referencedpath->{$path} = 1 if $path;
5821     }
5822
5823     foreach my $volid (sort keys %$volid_hash) {
5824         next if $volid =~ m/vm-$vmid-state-/;
5825         next if $referenced->{$volid};
5826         my $path = $volid_hash->{$volid}->{path};
5827         next if !$path; # just to be sure
5828         next if $referencedpath->{$path};
5829         $changes = 1;
5830         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5831         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5832         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5833     }
5834
5835     return $changes;
5836 }
5837
5838 sub rescan {
5839     my ($vmid, $nolock, $dryrun) = @_;
5840
5841     my $cfg = PVE::Storage::config();
5842
5843     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5844     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5845     foreach my $stor (keys %{$cfg->{ids}}) {
5846         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5847     }
5848
5849     print "rescan volumes...\n";
5850     my $volid_hash = scan_volids($cfg, $vmid);
5851
5852     my $updatefn =  sub {
5853         my ($vmid) = @_;
5854
5855         my $conf = PVE::QemuConfig->load_config($vmid);
5856
5857         PVE::QemuConfig->check_lock($conf);
5858
5859         my $vm_volids = {};
5860         foreach my $volid (keys %$volid_hash) {
5861             my $info = $volid_hash->{$volid};
5862             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5863         }
5864
5865         my $changes = update_disksize($vmid, $conf, $vm_volids);
5866
5867         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5868     };
5869
5870     if (defined($vmid)) {
5871         if ($nolock) {
5872             &$updatefn($vmid);
5873         } else {
5874             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5875         }
5876     } else {
5877         my $vmlist = config_list();
5878         foreach my $vmid (keys %$vmlist) {
5879             if ($nolock) {
5880                 &$updatefn($vmid);
5881             } else {
5882                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5883             }
5884         }
5885     }
5886 }
5887
5888 sub restore_vma_archive {
5889     my ($archive, $vmid, $user, $opts, $comp) = @_;
5890
5891     my $readfrom = $archive;
5892
5893     my $cfg = PVE::Storage::config();
5894     my $commands = [];
5895     my $bwlimit = $opts->{bwlimit};
5896
5897     my $dbg_cmdstring = '';
5898     my $add_pipe = sub {
5899         my ($cmd) = @_;
5900         push @$commands, $cmd;
5901         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5902         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5903         $readfrom = '-';
5904     };
5905
5906     my $input = undef;
5907     if ($archive eq '-') {
5908         $input = '<&STDIN';
5909     } else {
5910         # If we use a backup from a PVE defined storage we also consider that
5911         # storage's rate limit:
5912         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5913         if (defined($volid)) {
5914             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5915             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5916             if ($readlimit) {
5917                 print STDERR "applying read rate limit: $readlimit\n";
5918                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5919                 $add_pipe->($cstream);
5920             }
5921         }
5922     }
5923
5924     if ($comp) {
5925         my $cmd;
5926         if ($comp eq 'gzip') {
5927             $cmd = ['zcat', $readfrom];
5928         } elsif ($comp eq 'lzop') {
5929             $cmd = ['lzop', '-d', '-c', $readfrom];
5930         } else {
5931             die "unknown compression method '$comp'\n";
5932         }
5933         $add_pipe->($cmd);
5934     }
5935
5936     my $tmpdir = "/var/tmp/vzdumptmp$$";
5937     rmtree $tmpdir;
5938
5939     # disable interrupts (always do cleanups)
5940     local $SIG{INT} =
5941         local $SIG{TERM} =
5942         local $SIG{QUIT} =
5943         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5944
5945     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5946     POSIX::mkfifo($mapfifo, 0600);
5947     my $fifofh;
5948
5949     my $openfifo = sub {
5950         open($fifofh, '>', $mapfifo) || die $!;
5951     };
5952
5953     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5954
5955     my $oldtimeout;
5956     my $timeout = 5;
5957
5958     my $devinfo = {};
5959
5960     my $rpcenv = PVE::RPCEnvironment::get();
5961
5962     my $conffile = PVE::QemuConfig->config_file($vmid);
5963     my $tmpfn = "$conffile.$$.tmp";
5964
5965     # Note: $oldconf is undef if VM does not exists
5966     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5967     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5968
5969     my %storage_limits;
5970
5971     my $print_devmap = sub {
5972         my $virtdev_hash = {};
5973
5974         my $cfgfn = "$tmpdir/qemu-server.conf";
5975
5976         # we can read the config - that is already extracted
5977         my $fh = IO::File->new($cfgfn, "r") ||
5978             "unable to read qemu-server.conf - $!\n";
5979
5980         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5981         if (-f $fwcfgfn) {
5982             my $pve_firewall_dir = '/etc/pve/firewall';
5983             mkdir $pve_firewall_dir; # make sure the dir exists
5984             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5985         }
5986
5987         while (defined(my $line = <$fh>)) {
5988             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5989                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5990                 die "archive does not contain data for drive '$virtdev'\n"
5991                     if !$devinfo->{$devname};
5992                 if (defined($opts->{storage})) {
5993                     $storeid = $opts->{storage} || 'local';
5994                 } elsif (!$storeid) {
5995                     $storeid = 'local';
5996                 }
5997                 $format = 'raw' if !$format;
5998                 $devinfo->{$devname}->{devname} = $devname;
5999                 $devinfo->{$devname}->{virtdev} = $virtdev;
6000                 $devinfo->{$devname}->{format} = $format;
6001                 $devinfo->{$devname}->{storeid} = $storeid;
6002
6003                 # check permission on storage
6004                 my $pool = $opts->{pool}; # todo: do we need that?
6005                 if ($user ne 'root@pam') {
6006                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6007                 }
6008
6009                 $storage_limits{$storeid} = $bwlimit;
6010
6011                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6012             }
6013         }
6014
6015         foreach my $key (keys %storage_limits) {
6016             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6017             next if !$limit;
6018             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6019             $storage_limits{$key} = $limit * 1024;
6020         }
6021
6022         foreach my $devname (keys %$devinfo) {
6023             die "found no device mapping information for device '$devname'\n"
6024                 if !$devinfo->{$devname}->{virtdev};
6025         }
6026
6027         # create empty/temp config
6028         if ($oldconf) {
6029             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6030             foreach_drive($oldconf, sub {
6031                 my ($ds, $drive) = @_;
6032
6033                 return if drive_is_cdrom($drive);
6034
6035                 my $volid = $drive->{file};
6036
6037                 return if !$volid || $volid =~ m|^/|;
6038
6039                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6040                 return if !$path || !$owner || ($owner != $vmid);
6041
6042                 # Note: only delete disk we want to restore
6043                 # other volumes will become unused
6044                 if ($virtdev_hash->{$ds}) {
6045                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6046                     if (my $err = $@) {
6047                         warn $err;
6048                     }
6049                 }
6050             });
6051
6052             # delete vmstate files
6053             # since after the restore we have no snapshots anymore
6054             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6055                 my $snap = $oldconf->{snapshots}->{$snapname};
6056                 if ($snap->{vmstate}) {
6057                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6058                     if (my $err = $@) {
6059                         warn $err;
6060                     }
6061                 }
6062             }
6063         }
6064
6065         my $map = {};
6066         foreach my $virtdev (sort keys %$virtdev_hash) {
6067             my $d = $virtdev_hash->{$virtdev};
6068             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6069             my $storeid = $d->{storeid};
6070             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6071
6072             my $map_opts = '';
6073             if (my $limit = $storage_limits{$storeid}) {
6074                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6075             }
6076
6077             # test if requested format is supported
6078             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6079             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6080             $d->{format} = $defFormat if !$supported;
6081
6082             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6083                                                   $d->{format}, undef, $alloc_size);
6084             print STDERR "new volume ID is '$volid'\n";
6085             $d->{volid} = $volid;
6086             my $path = PVE::Storage::path($cfg, $volid);
6087
6088             PVE::Storage::activate_volumes($cfg,[$volid]);
6089
6090             my $write_zeros = 1;
6091             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6092                 $write_zeros = 0;
6093             }
6094
6095             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6096
6097             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6098             $map->{$virtdev} = $volid;
6099         }
6100
6101         $fh->seek(0, 0) || die "seek failed - $!\n";
6102
6103         my $outfd = new IO::File ($tmpfn, "w") ||
6104             die "unable to write config for VM $vmid\n";
6105
6106         my $cookie = { netcount => 0 };
6107         while (defined(my $line = <$fh>)) {
6108             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6109         }
6110
6111         $fh->close();
6112         $outfd->close();
6113     };
6114
6115     eval {
6116         # enable interrupts
6117         local $SIG{INT} =
6118             local $SIG{TERM} =
6119             local $SIG{QUIT} =
6120             local $SIG{HUP} =
6121             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6122         local $SIG{ALRM} = sub { die "got timeout\n"; };
6123
6124         $oldtimeout = alarm($timeout);
6125
6126         my $parser = sub {
6127             my $line = shift;
6128
6129             print "$line\n";
6130
6131             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6132                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6133                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6134             } elsif ($line =~ m/^CTIME: /) {
6135                 # we correctly received the vma config, so we can disable
6136                 # the timeout now for disk allocation (set to 10 minutes, so
6137                 # that we always timeout if something goes wrong)
6138                 alarm(600);
6139                 &$print_devmap();
6140                 print $fifofh "done\n";
6141                 my $tmp = $oldtimeout || 0;
6142                 $oldtimeout = undef;
6143                 alarm($tmp);
6144                 close($fifofh);
6145             }
6146         };
6147
6148         print "restore vma archive: $dbg_cmdstring\n";
6149         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6150     };
6151     my $err = $@;
6152
6153     alarm($oldtimeout) if $oldtimeout;
6154
6155     my $vollist = [];
6156     foreach my $devname (keys %$devinfo) {
6157         my $volid = $devinfo->{$devname}->{volid};
6158         push @$vollist, $volid if $volid;
6159     }
6160
6161     PVE::Storage::deactivate_volumes($cfg, $vollist);
6162
6163     unlink $mapfifo;
6164
6165     if ($err) {
6166         rmtree $tmpdir;
6167         unlink $tmpfn;
6168
6169         foreach my $devname (keys %$devinfo) {
6170             my $volid = $devinfo->{$devname}->{volid};
6171             next if !$volid;
6172             eval {
6173                 if ($volid =~ m|^/|) {
6174                     unlink $volid || die 'unlink failed\n';
6175                 } else {
6176                     PVE::Storage::vdisk_free($cfg, $volid);
6177                 }
6178                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6179             };
6180             print STDERR "unable to cleanup '$volid' - $@" if $@;
6181         }
6182         die $err;
6183     }
6184
6185     rmtree $tmpdir;
6186
6187     rename($tmpfn, $conffile) ||
6188         die "unable to commit configuration file '$conffile'\n";
6189
6190     PVE::Cluster::cfs_update(); # make sure we read new file
6191
6192     eval { rescan($vmid, 1); };
6193     warn $@ if $@;
6194 }
6195
6196 sub restore_tar_archive {
6197     my ($archive, $vmid, $user, $opts) = @_;
6198
6199     if ($archive ne '-') {
6200         my $firstfile = tar_archive_read_firstfile($archive);
6201         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6202             if $firstfile ne 'qemu-server.conf';
6203     }
6204
6205     my $storecfg = PVE::Storage::config();
6206
6207     # destroy existing data - keep empty config
6208     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6209     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6210
6211     my $tocmd = "/usr/lib/qemu-server/qmextract";
6212
6213     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6214     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6215     $tocmd .= ' --prealloc' if $opts->{prealloc};
6216     $tocmd .= ' --info' if $opts->{info};
6217
6218     # tar option "xf" does not autodetect compression when read from STDIN,
6219     # so we pipe to zcat
6220     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6221         PVE::Tools::shellquote("--to-command=$tocmd");
6222
6223     my $tmpdir = "/var/tmp/vzdumptmp$$";
6224     mkpath $tmpdir;
6225
6226     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6227     local $ENV{VZDUMP_VMID} = $vmid;
6228     local $ENV{VZDUMP_USER} = $user;
6229
6230     my $conffile = PVE::QemuConfig->config_file($vmid);
6231     my $tmpfn = "$conffile.$$.tmp";
6232
6233     # disable interrupts (always do cleanups)
6234     local $SIG{INT} =
6235         local $SIG{TERM} =
6236         local $SIG{QUIT} =
6237         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6238
6239     eval {
6240         # enable interrupts
6241         local $SIG{INT} =
6242             local $SIG{TERM} =
6243             local $SIG{QUIT} =
6244             local $SIG{HUP} =
6245             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6246
6247         if ($archive eq '-') {
6248             print "extracting archive from STDIN\n";
6249             run_command($cmd, input => "<&STDIN");
6250         } else {
6251             print "extracting archive '$archive'\n";
6252             run_command($cmd);
6253         }
6254
6255         return if $opts->{info};
6256
6257         # read new mapping
6258         my $map = {};
6259         my $statfile = "$tmpdir/qmrestore.stat";
6260         if (my $fd = IO::File->new($statfile, "r")) {
6261             while (defined (my $line = <$fd>)) {
6262                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6263                     $map->{$1} = $2 if $1;
6264                 } else {
6265                     print STDERR "unable to parse line in statfile - $line\n";
6266                 }
6267             }
6268             $fd->close();
6269         }
6270
6271         my $confsrc = "$tmpdir/qemu-server.conf";
6272
6273         my $srcfd = new IO::File($confsrc, "r") ||
6274             die "unable to open file '$confsrc'\n";
6275
6276         my $outfd = new IO::File ($tmpfn, "w") ||
6277             die "unable to write config for VM $vmid\n";
6278
6279         my $cookie = { netcount => 0 };
6280         while (defined (my $line = <$srcfd>)) {
6281             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6282         }
6283
6284         $srcfd->close();
6285         $outfd->close();
6286     };
6287     my $err = $@;
6288
6289     if ($err) {
6290
6291         unlink $tmpfn;
6292
6293         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6294
6295         die $err;
6296     }
6297
6298     rmtree $tmpdir;
6299
6300     rename $tmpfn, $conffile ||
6301         die "unable to commit configuration file '$conffile'\n";
6302
6303     PVE::Cluster::cfs_update(); # make sure we read new file
6304
6305     eval { rescan($vmid, 1); };
6306     warn $@ if $@;
6307 };
6308
6309 sub foreach_storage_used_by_vm {
6310     my ($conf, $func) = @_;
6311
6312     my $sidhash = {};
6313
6314     foreach_drive($conf, sub {
6315         my ($ds, $drive) = @_;
6316         return if drive_is_cdrom($drive);
6317
6318         my $volid = $drive->{file};
6319
6320         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6321         $sidhash->{$sid} = $sid if $sid;
6322     });
6323
6324     foreach my $sid (sort keys %$sidhash) {
6325         &$func($sid);
6326     }
6327 }
6328
6329 sub do_snapshots_with_qemu {
6330     my ($storecfg, $volid) = @_;
6331
6332     my $storage_name = PVE::Storage::parse_volume_id($volid);
6333
6334     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6335         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6336         return 1;
6337     }
6338
6339     if ($volid =~ m/\.(qcow2|qed)$/){
6340         return 1;
6341     }
6342
6343     return undef;
6344 }
6345
6346 sub qga_check_running {
6347     my ($vmid, $nowarn) = @_;
6348
6349     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6350     if ($@) {
6351         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6352         return 0;
6353     }
6354     return 1;
6355 }
6356
6357 sub template_create {
6358     my ($vmid, $conf, $disk) = @_;
6359
6360     my $storecfg = PVE::Storage::config();
6361
6362     foreach_drive($conf, sub {
6363         my ($ds, $drive) = @_;
6364
6365         return if drive_is_cdrom($drive);
6366         return if $disk && $ds ne $disk;
6367
6368         my $volid = $drive->{file};
6369         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6370
6371         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6372         $drive->{file} = $voliddst;
6373         $conf->{$ds} = print_drive($vmid, $drive);
6374         PVE::QemuConfig->write_config($vmid, $conf);
6375     });
6376 }
6377
6378 sub qemu_img_convert {
6379     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6380
6381     my $storecfg = PVE::Storage::config();
6382     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6383     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6384
6385     if ($src_storeid && $dst_storeid) {
6386
6387         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6388
6389         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6390         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6391
6392         my $src_format = qemu_img_format($src_scfg, $src_volname);
6393         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6394
6395         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6396         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6397
6398         my $cmd = [];
6399         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6400         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6401         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6402         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6403         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6404         if ($is_zero_initialized) {
6405             push @$cmd, "zeroinit:$dst_path";
6406         } else {
6407             push @$cmd, $dst_path;
6408         }
6409
6410         my $parser = sub {
6411             my $line = shift;
6412             if($line =~ m/\((\S+)\/100\%\)/){
6413                 my $percent = $1;
6414                 my $transferred = int($size * $percent / 100);
6415                 my $remaining = $size - $transferred;
6416
6417                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6418             }
6419
6420         };
6421
6422         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6423         my $err = $@;
6424         die "copy failed: $err" if $err;
6425     }
6426 }
6427
6428 sub qemu_img_format {
6429     my ($scfg, $volname) = @_;
6430
6431     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6432         return $1;
6433     } else {
6434         return "raw";
6435     }
6436 }
6437
6438 sub qemu_drive_mirror {
6439     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6440
6441     $jobs = {} if !$jobs;
6442
6443     my $qemu_target;
6444     my $format;
6445     $jobs->{"drive-$drive"} = {};
6446
6447     if ($dst_volid =~ /^nbd:/) {
6448         $qemu_target = $dst_volid;
6449         $format = "nbd";
6450     } else {
6451         my $storecfg = PVE::Storage::config();
6452         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6453
6454         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6455
6456         $format = qemu_img_format($dst_scfg, $dst_volname);
6457
6458         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6459
6460         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6461     }
6462
6463     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6464     $opts->{format} = $format if $format;
6465
6466     print "drive mirror is starting for drive-$drive\n";
6467
6468     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6469
6470     if (my $err = $@) {
6471         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6472         die "mirroring error: $err";
6473     }
6474
6475     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6476 }
6477
6478 sub qemu_drive_mirror_monitor {
6479     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6480
6481     eval {
6482         my $err_complete = 0;
6483
6484         while (1) {
6485             die "storage migration timed out\n" if $err_complete > 300;
6486
6487             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6488
6489             my $running_mirror_jobs = {};
6490             foreach my $stat (@$stats) {
6491                 next if $stat->{type} ne 'mirror';
6492                 $running_mirror_jobs->{$stat->{device}} = $stat;
6493             }
6494
6495             my $readycounter = 0;
6496
6497             foreach my $job (keys %$jobs) {
6498
6499                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6500                     print "$job : finished\n";
6501                     delete $jobs->{$job};
6502                     next;
6503                 }
6504
6505                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6506
6507                 my $busy = $running_mirror_jobs->{$job}->{busy};
6508                 my $ready = $running_mirror_jobs->{$job}->{ready};
6509                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6510                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6511                     my $remaining = $total - $transferred;
6512                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6513
6514                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6515                 }
6516
6517                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6518             }
6519
6520             last if scalar(keys %$jobs) == 0;
6521
6522             if ($readycounter == scalar(keys %$jobs)) {
6523                 print "all mirroring jobs are ready \n";
6524                 last if $skipcomplete; #do the complete later
6525
6526                 if ($vmiddst && $vmiddst != $vmid) {
6527                     my $agent_running = $qga && qga_check_running($vmid);
6528                     if ($agent_running) {
6529                         print "freeze filesystem\n";
6530                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6531                     } else {
6532                         print "suspend vm\n";
6533                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6534                     }
6535
6536                     # if we clone a disk for a new target vm, we don't switch the disk
6537                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6538
6539                     if ($agent_running) {
6540                         print "unfreeze filesystem\n";
6541                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6542                     } else {
6543                         print "resume vm\n";
6544                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6545                     }
6546
6547                     last;
6548                 } else {
6549
6550                     foreach my $job (keys %$jobs) {
6551                         # try to switch the disk if source and destination are on the same guest
6552                         print "$job: Completing block job...\n";
6553
6554                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6555                         if ($@ =~ m/cannot be completed/) {
6556                             print "$job: Block job cannot be completed, try again.\n";
6557                             $err_complete++;
6558                         }else {
6559                             print "$job: Completed successfully.\n";
6560                             $jobs->{$job}->{complete} = 1;
6561                         }
6562                     }
6563                 }
6564             }
6565             sleep 1;
6566         }
6567     };
6568     my $err = $@;
6569
6570     if ($err) {
6571         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6572         die "mirroring error: $err";
6573     }
6574
6575 }
6576
6577 sub qemu_blockjobs_cancel {
6578     my ($vmid, $jobs) = @_;
6579
6580     foreach my $job (keys %$jobs) {
6581         print "$job: Cancelling block job\n";
6582         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6583         $jobs->{$job}->{cancel} = 1;
6584     }
6585
6586     while (1) {
6587         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6588
6589         my $running_jobs = {};
6590         foreach my $stat (@$stats) {
6591             $running_jobs->{$stat->{device}} = $stat;
6592         }
6593
6594         foreach my $job (keys %$jobs) {
6595
6596             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6597                 print "$job: Done.\n";
6598                 delete $jobs->{$job};
6599             }
6600         }
6601
6602         last if scalar(keys %$jobs) == 0;
6603
6604         sleep 1;
6605     }
6606 }
6607
6608 sub clone_disk {
6609     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6610         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6611
6612     my $newvolid;
6613
6614     if (!$full) {
6615         print "create linked clone of drive $drivename ($drive->{file})\n";
6616         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6617         push @$newvollist, $newvolid;
6618     } else {
6619
6620         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6621         $storeid = $storage if $storage;
6622
6623         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6624         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6625
6626         print "create full clone of drive $drivename ($drive->{file})\n";
6627         my $name = undef;
6628         if (drive_is_cloudinit($drive)) {
6629             $name = "vm-$newvmid-cloudinit";
6630             # cloudinit only supports raw and qcow2 atm:
6631             if ($dst_format eq 'qcow2') {
6632                 $name .= '.qcow2';
6633             } elsif ($dst_format ne 'raw') {
6634                 die "clone: unhandled format for cloudinit image\n";
6635             }
6636         }
6637         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6638         push @$newvollist, $newvolid;
6639
6640         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6641
6642         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6643         if (!$running || $snapname) {
6644             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6645         } else {
6646
6647             my $kvmver = get_running_qemu_version ($vmid);
6648             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6649                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6650                     if $drive->{iothread};
6651             }
6652
6653             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6654         }
6655     }
6656
6657     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6658
6659     my $disk = $drive;
6660     $disk->{format} = undef;
6661     $disk->{file} = $newvolid;
6662     $disk->{size} = $size;
6663
6664     return $disk;
6665 }
6666
6667 # this only works if VM is running
6668 sub get_current_qemu_machine {
6669     my ($vmid) = @_;
6670
6671     my $cmd = { execute => 'query-machines', arguments => {} };
6672     my $res = vm_qmp_command($vmid, $cmd);
6673
6674     my ($current, $default);
6675     foreach my $e (@$res) {
6676         $default = $e->{name} if $e->{'is-default'};
6677         $current = $e->{name} if $e->{'is-current'};
6678     }
6679
6680     # fallback to the default machine if current is not supported by qemu
6681     return $current || $default || 'pc';
6682 }
6683
6684 sub get_running_qemu_version {
6685     my ($vmid) = @_;
6686     my $cmd = { execute => 'query-version', arguments => {} };
6687     my $res = vm_qmp_command($vmid, $cmd);
6688     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6689 }
6690
6691 sub qemu_machine_feature_enabled {
6692     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6693
6694     my $current_major;
6695     my $current_minor;
6696
6697     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6698
6699         $current_major = $3;
6700         $current_minor = $4;
6701
6702     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6703
6704         $current_major = $1;
6705         $current_minor = $2;
6706     }
6707
6708     return 1 if $current_major > $version_major ||
6709                 ($current_major == $version_major &&
6710                  $current_minor >= $version_minor);
6711 }
6712
6713 sub qemu_machine_pxe {
6714     my ($vmid, $conf, $machine) = @_;
6715
6716     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6717
6718     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6719         $machine .= '.pxe';
6720     }
6721
6722     return $machine;
6723 }
6724
6725 sub qemu_use_old_bios_files {
6726     my ($machine_type) = @_;
6727
6728     return if !$machine_type;
6729
6730     my $use_old_bios_files = undef;
6731
6732     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6733         $machine_type = $1;
6734         $use_old_bios_files = 1;
6735     } else {
6736         my $kvmver = kvm_user_version();
6737         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6738         # load new efi bios files on migration. So this hack is required to allow
6739         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6740         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6741         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6742     }
6743
6744     return ($use_old_bios_files, $machine_type);
6745 }
6746
6747 sub create_efidisk {
6748     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6749
6750     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6751
6752     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6753     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6754     PVE::Storage::activate_volumes($storecfg, [$volid]);
6755
6756     my $path = PVE::Storage::path($storecfg, $volid);
6757     eval {
6758         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6759     };
6760     die "Copying EFI vars image failed: $@" if $@;
6761
6762     return ($volid, $vars_size);
6763 }
6764
6765 sub lspci {
6766
6767     my $devices = {};
6768
6769     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6770             my (undef, $id, $function) = @_;
6771             my $res = { id => $id, function => $function};
6772             push @{$devices->{$id}}, $res;
6773     });
6774
6775     # Entries should be sorted by functions.
6776     foreach my $id (keys %$devices) {
6777         my $dev = $devices->{$id};
6778         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6779     }
6780
6781     return $devices;
6782 }
6783
6784 sub vm_iothreads_list {
6785     my ($vmid) = @_;
6786
6787     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6788
6789     my $iothreads = {};
6790     foreach my $iothread (@$res) {
6791         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6792     }
6793
6794     return $iothreads;
6795 }
6796
6797 sub scsihw_infos {
6798     my ($conf, $drive) = @_;
6799
6800     my $maxdev = 0;
6801
6802     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6803         $maxdev = 7;
6804     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6805         $maxdev = 1;
6806     } else {
6807         $maxdev = 256;
6808     }
6809
6810     my $controller = int($drive->{index} / $maxdev);
6811     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6812
6813     return ($maxdev, $controller, $controller_prefix);
6814 }
6815
6816 sub add_hyperv_enlightenments {
6817     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6818
6819     return if $winversion < 6;
6820     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6821
6822     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6823
6824     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6825         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6826         push @$cpuFlags , 'hv_vapic';
6827         push @$cpuFlags , 'hv_time';
6828     } else {
6829         push @$cpuFlags , 'hv_spinlocks=0xffff';
6830     }
6831
6832     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6833         push @$cpuFlags , 'hv_reset';
6834         push @$cpuFlags , 'hv_vpindex';
6835         push @$cpuFlags , 'hv_runtime';
6836     }
6837
6838     if ($winversion >= 7) {
6839         push @$cpuFlags , 'hv_relaxed';
6840
6841         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6842             push @$cpuFlags , 'hv_synic';
6843             push @$cpuFlags , 'hv_stimer';
6844         }
6845     }
6846 }
6847
6848 sub windows_version {
6849     my ($ostype) = @_;
6850
6851     return 0 if !$ostype;
6852
6853     my $winversion = 0;
6854
6855     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6856         $winversion = 5;
6857     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6858         $winversion = 6;
6859     } elsif ($ostype =~ m/^win(\d+)$/) {
6860         $winversion = $1;
6861     }
6862
6863     return $winversion;
6864 }
6865
6866 sub resolve_dst_disk_format {
6867         my ($storecfg, $storeid, $src_volname, $format) = @_;
6868         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6869
6870         if (!$format) {
6871             # if no target format is specified, use the source disk format as hint
6872             if ($src_volname) {
6873                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6874                 $format = qemu_img_format($scfg, $src_volname);
6875             } else {
6876                 return $defFormat;
6877             }
6878         }
6879
6880         # test if requested format is supported - else use default
6881         my $supported = grep { $_ eq $format } @$validFormats;
6882         $format = $defFormat if !$supported;
6883         return $format;
6884 }
6885
6886 sub resolve_first_disk {
6887     my $conf = shift;
6888     my @disks = PVE::QemuServer::valid_drive_names();
6889     my $firstdisk;
6890     foreach my $ds (reverse @disks) {
6891         next if !$conf->{$ds};
6892         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6893         next if PVE::QemuServer::drive_is_cdrom($disk);
6894         $firstdisk = $ds;
6895     }
6896     return $firstdisk;
6897 }
6898
6899 sub generate_uuid {
6900     my ($uuid, $uuid_str);
6901     UUID::generate($uuid);
6902     UUID::unparse($uuid, $uuid_str);
6903     return $uuid_str;
6904 }
6905
6906 sub generate_smbios1_uuid {
6907     return "uuid=".generate_uuid();
6908 }
6909
6910 sub nbd_stop {
6911     my ($vmid) = @_;
6912
6913     vm_mon_cmd($vmid, 'nbd-server-stop');
6914 }
6915
6916 # bash completion helper
6917
6918 sub complete_backup_archives {
6919     my ($cmdname, $pname, $cvalue) = @_;
6920
6921     my $cfg = PVE::Storage::config();
6922
6923     my $storeid;
6924
6925     if ($cvalue =~ m/^([^:]+):/) {
6926         $storeid = $1;
6927     }
6928
6929     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6930
6931     my $res = [];
6932     foreach my $id (keys %$data) {
6933         foreach my $item (@{$data->{$id}}) {
6934             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6935             push @$res, $item->{volid} if defined($item->{volid});
6936         }
6937     }
6938
6939     return $res;
6940 }
6941
6942 my $complete_vmid_full = sub {
6943     my ($running) = @_;
6944
6945     my $idlist = vmstatus();
6946
6947     my $res = [];
6948
6949     foreach my $id (keys %$idlist) {
6950         my $d = $idlist->{$id};
6951         if (defined($running)) {
6952             next if $d->{template};
6953             next if $running && $d->{status} ne 'running';
6954             next if !$running && $d->{status} eq 'running';
6955         }
6956         push @$res, $id;
6957
6958     }
6959     return $res;
6960 };
6961
6962 sub complete_vmid {
6963     return &$complete_vmid_full();
6964 }
6965
6966 sub complete_vmid_stopped {
6967     return &$complete_vmid_full(0);
6968 }
6969
6970 sub complete_vmid_running {
6971     return &$complete_vmid_full(1);
6972 }
6973
6974 sub complete_storage {
6975
6976     my $cfg = PVE::Storage::config();
6977     my $ids = $cfg->{ids};
6978
6979     my $res = [];
6980     foreach my $sid (keys %$ids) {
6981         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6982         next if !$ids->{$sid}->{content}->{images};
6983         push @$res, $sid;
6984     }
6985
6986     return $res;
6987 }
6988
6989 1;