PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF = {
  44     x86_64 => [
  45         "$EDK2_FW_BASE/OVMF_CODE.fd",
  46         "$EDK2_FW_BASE/OVMF_VARS.fd"
  47     ],
  48     aarch64 => [
  49         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  50         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  51     ],
  52 };
  53
  54 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  55
  56 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  57
  58 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  59
  60 # Note about locking: we use flock on the config file protect
  61 # against concurent actions.
  62 # Aditionaly, we have a 'lock' setting in the config file. This
  63 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  64 # allowed when such lock is set. But you can ignore this kind of
  65 # lock with the --skiplock flag.
  66
  67 cfs_register_file('/qemu-server/',
  68                   \&parse_vm_config,
  69                   \&write_vm_config);
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  72     description => "Some command save/restore state from this location.",
  73     type => 'string',
  74     maxLength => 128,
  75     optional => 1,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  79     description => "The name of the snapshot.",
  80     type => 'string', format => 'pve-configid',
  81     maxLength => 40,
  82 });
  83
  84 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  85     type => 'string',
  86     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  87     description => "The drive's backing file's data format.",
  88     optional => 1,
  89 });
  90
  91 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  92         description => "Specifies the Qemu machine type.",
  93         type => 'string',
  94         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  95         maxLength => 40,
  96         optional => 1,
  97 });
  98
  99 #no warnings 'redefine';
 100
 101 sub cgroups_write {
 102    my ($controller, $vmid, $option, $value) = @_;
 103
 104    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 105    PVE::ProcFSTools::write_proc_entry($path, $value);
 106
 107 }
 108
 109 my $nodename = PVE::INotify::nodename();
 110
 111 mkdir "/etc/pve/nodes/$nodename";
 112 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 113 mkdir $confdir;
 114
 115 my $var_run_tmpdir = "/var/run/qemu-server";
 116 mkdir $var_run_tmpdir;
 117
 118 my $lock_dir = "/var/lock/qemu-server";
 119 mkdir $lock_dir;
 120
 121 my $pcisysfs = "/sys/bus/pci";
 122
 123 my $cpu_vendor_list = {
 124     # Intel CPUs
 125     486 => 'GenuineIntel',
 126     pentium => 'GenuineIntel',
 127     pentium2  => 'GenuineIntel',
 128     pentium3  => 'GenuineIntel',
 129     coreduo => 'GenuineIntel',
 130     core2duo => 'GenuineIntel',
 131     Conroe  => 'GenuineIntel',
 132     Penryn  => 'GenuineIntel',
 133     Nehalem  => 'GenuineIntel',
 134     'Nehalem-IBRS'  => 'GenuineIntel',
 135     Westmere => 'GenuineIntel',
 136     'Westmere-IBRS' => 'GenuineIntel',
 137     SandyBridge => 'GenuineIntel',
 138     'SandyBridge-IBRS' => 'GenuineIntel',
 139     IvyBridge => 'GenuineIntel',
 140     'IvyBridge-IBRS' => 'GenuineIntel',
 141     Haswell => 'GenuineIntel',
 142     'Haswell-IBRS' => 'GenuineIntel',
 143     'Haswell-noTSX' => 'GenuineIntel',
 144     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 145     Broadwell => 'GenuineIntel',
 146     'Broadwell-IBRS' => 'GenuineIntel',
 147     'Broadwell-noTSX' => 'GenuineIntel',
 148     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 149     'Skylake-Client' => 'GenuineIntel',
 150     'Skylake-Client-IBRS' => 'GenuineIntel',
 151     'Skylake-Server' => 'GenuineIntel',
 152     'Skylake-Server-IBRS' => 'GenuineIntel',
 153
 154     # AMD CPUs
 155     athlon => 'AuthenticAMD',
 156     phenom  => 'AuthenticAMD',
 157     Opteron_G1  => 'AuthenticAMD',
 158     Opteron_G2  => 'AuthenticAMD',
 159     Opteron_G3  => 'AuthenticAMD',
 160     Opteron_G4  => 'AuthenticAMD',
 161     Opteron_G5  => 'AuthenticAMD',
 162     EPYC => 'AuthenticAMD',
 163     'EPYC-IBPB' => 'AuthenticAMD',
 164
 165     # generic types, use vendor from host node
 166     host => 'default',
 167     kvm32 => 'default',
 168     kvm64 => 'default',
 169     qemu32 => 'default',
 170     qemu64 => 'default',
 171     max => 'default',
 172 };
 173
 174 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 175
 176 my $cpu_fmt = {
 177     cputype => {
 178         description => "Emulated CPU type.",
 179         type => 'string',
 180         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 181         default => 'kvm64',
 182         default_key => 1,
 183     },
 184     hidden => {
 185         description => "Do not identify as a KVM virtual machine.",
 186         type => 'boolean',
 187         optional => 1,
 188         default => 0
 189     },
 190     flags => {
 191         description => "List of additional CPU flags separated by ';'."
 192                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 193                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 194         format_description => '+FLAG[;-FLAG...]',
 195         type => 'string',
 196         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 197         optional => 1,
 198     },
 199 };
 200
 201 my $watchdog_fmt = {
 202     model => {
 203         default_key => 1,
 204         type => 'string',
 205         enum => [qw(i6300esb ib700)],
 206         description => "Watchdog type to emulate.",
 207         default => 'i6300esb',
 208         optional => 1,
 209     },
 210     action => {
 211         type => 'string',
 212         enum => [qw(reset shutdown poweroff pause debug none)],
 213         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 214         optional => 1,
 215     },
 216 };
 217 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 218
 219 my $agent_fmt = {
 220     enabled => {
 221         description => "Enable/disable Qemu GuestAgent.",
 222         type => 'boolean',
 223         default => 0,
 224         default_key => 1,
 225     },
 226     fstrim_cloned_disks => {
 227         description => "Run fstrim after cloning/moving a disk.",
 228         type => 'boolean',
 229         optional => 1,
 230         default => 0
 231     },
 232 };
 233
 234 my $vga_fmt = {
 235     type => {
 236         description => "Select the VGA type.",
 237         type => 'string',
 238         default => 'std',
 239         optional => 1,
 240         default_key => 1,
 241         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 242     },
 243     memory => {
 244         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 245         type => 'integer',
 246         optional => 1,
 247         minimum => 4,
 248         maximum => 512,
 249     },
 250 };
 251
 252 my $confdesc = {
 253     onboot => {
 254         optional => 1,
 255         type => 'boolean',
 256         description => "Specifies whether a VM will be started during system bootup.",
 257         default => 0,
 258     },
 259     autostart => {
 260         optional => 1,
 261         type => 'boolean',
 262         description => "Automatic restart after crash (currently ignored).",
 263         default => 0,
 264     },
 265     hotplug => {
 266         optional => 1,
 267         type => 'string', format => 'pve-hotplug-features',
 268         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 269         default => 'network,disk,usb',
 270     },
 271     reboot => {
 272         optional => 1,
 273         type => 'boolean',
 274         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 275         default => 1,
 276     },
 277     lock => {
 278         optional => 1,
 279         type => 'string',
 280         description => "Lock/unlock the VM.",
 281         enum => [qw(migrate backup snapshot rollback)],
 282     },
 283     cpulimit => {
 284         optional => 1,
 285         type => 'number',
 286         description => "Limit of CPU usage.",
 287         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 288         minimum => 0,
 289         maximum => 128,
 290         default => 0,
 291     },
 292     cpuunits => {
 293         optional => 1,
 294         type => 'integer',
 295         description => "CPU weight for a VM.",
 296         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 297         minimum => 2,
 298         maximum => 262144,
 299         default => 1024,
 300     },
 301     memory => {
 302         optional => 1,
 303         type => 'integer',
 304         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 305         minimum => 16,
 306         default => 512,
 307     },
 308     balloon => {
 309         optional => 1,
 310         type => 'integer',
 311         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 312         minimum => 0,
 313     },
 314     shares => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 318         minimum => 0,
 319         maximum => 50000,
 320         default => 1000,
 321     },
 322     keyboard => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 326                        "It should not be necessary to set it.",
 327         enum => PVE::Tools::kvmkeymaplist(),
 328         default => undef,
 329     },
 330     name => {
 331         optional => 1,
 332         type => 'string', format => 'dns-name',
 333         description => "Set a name for the VM. Only used on the configuration web interface.",
 334     },
 335     scsihw => {
 336         optional => 1,
 337         type => 'string',
 338         description => "SCSI controller model",
 339         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 340         default => 'lsi',
 341     },
 342     description => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 346     },
 347     ostype => {
 348         optional => 1,
 349         type => 'string',
 350         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 351         description => "Specify guest operating system.",
 352         verbose_description => <<EODESC,
 353 Specify guest operating system. This is used to enable special
 354 optimization/features for specific operating systems:
 355
 356 [horizontal]
 357 other;; unspecified OS
 358 wxp;; Microsoft Windows XP
 359 w2k;; Microsoft Windows 2000
 360 w2k3;; Microsoft Windows 2003
 361 w2k8;; Microsoft Windows 2008
 362 wvista;; Microsoft Windows Vista
 363 win7;; Microsoft Windows 7
 364 win8;; Microsoft Windows 8/2012/2012r2
 365 win10;; Microsoft Windows 10/2016
 366 l24;; Linux 2.4 Kernel
 367 l26;; Linux 2.6/3.X Kernel
 368 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 369 EODESC
 370     },
 371     boot => {
 372         optional => 1,
 373         type => 'string',
 374         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 375         pattern => '[acdn]{1,4}',
 376         default => 'cdn',
 377     },
 378     bootdisk => {
 379         optional => 1,
 380         type => 'string', format => 'pve-qm-bootdisk',
 381         description => "Enable booting from specified disk.",
 382         pattern => '(ide|sata|scsi|virtio)\d+',
 383     },
 384     smp => {
 385         optional => 1,
 386         type => 'integer',
 387         description => "The number of CPUs. Please use option -sockets instead.",
 388         minimum => 1,
 389         default => 1,
 390     },
 391     sockets => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "The number of CPU sockets.",
 395         minimum => 1,
 396         default => 1,
 397     },
 398     cores => {
 399         optional => 1,
 400         type => 'integer',
 401         description => "The number of cores per socket.",
 402         minimum => 1,
 403         default => 1,
 404     },
 405     numa => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable NUMA.",
 409         default => 0,
 410     },
 411     hugepages => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Enable/disable hugepages memory.",
 415         enum => [qw(any 2 1024)],
 416     },
 417     vcpus => {
 418         optional => 1,
 419         type => 'integer',
 420         description => "Number of hotplugged vcpus.",
 421         minimum => 1,
 422         default => 0,
 423     },
 424     acpi => {
 425         optional => 1,
 426         type => 'boolean',
 427         description => "Enable/disable ACPI.",
 428         default => 1,
 429     },
 430     agent => {
 431         optional => 1,
 432         description => "Enable/disable Qemu GuestAgent and its properties.",
 433         type => 'string',
 434         format => $agent_fmt,
 435     },
 436     kvm => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable KVM hardware virtualization.",
 440         default => 1,
 441     },
 442     tdf => {
 443         optional => 1,
 444         type => 'boolean',
 445         description => "Enable/disable time drift fix.",
 446         default => 0,
 447     },
 448     localtime => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 452     },
 453     freeze => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 457     },
 458     vga => {
 459         optional => 1,
 460         type => 'string', format => $vga_fmt,
 461         description => "Configure the VGA hardware.",
 462         verbose_description => "Configure the VGA Hardware. If you want to use ".
 463             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 464             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 465             "is 'std' for all OS types besides some Windows versions (XP and " .
 466             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 467             "display server. For win* OS you can select how many independent " .
 468             "displays you want, Linux guests can add displays them self.\n".
 469             "You can also run without any graphic card, using a serial device as terminal.",
 470     },
 471     watchdog => {
 472         optional => 1,
 473         type => 'string', format => 'pve-qm-watchdog',
 474         description => "Create a virtual hardware watchdog device.",
 475         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 476             " (by a guest action), the watchdog must be periodically polled " .
 477             "by an agent inside the guest or else the watchdog will reset " .
 478             "the guest (or execute the respective action specified)",
 479     },
 480     startdate => {
 481         optional => 1,
 482         type => 'string',
 483         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 484         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 485         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 486         default => 'now',
 487     },
 488     startup =>  get_standard_option('pve-startup-order'),
 489     template => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable Template.",
 493         default => 0,
 494     },
 495     args => {
 496         optional => 1,
 497         type => 'string',
 498         description => "Arbitrary arguments passed to kvm.",
 499         verbose_description => <<EODESCR,
 500 Arbitrary arguments passed to kvm, for example:
 501
 502 args: -no-reboot -no-hpet
 503
 504 NOTE: this option is for experts only.
 505 EODESCR
 506     },
 507     tablet => {
 508         optional => 1,
 509         type => 'boolean',
 510         default => 1,
 511         description => "Enable/disable the USB tablet device.",
 512         verbose_description => "Enable/disable the USB tablet device. This device is " .
 513             "usually needed to allow absolute mouse positioning with VNC. " .
 514             "Else the mouse runs out of sync with normal VNC clients. " .
 515             "If you're running lots of console-only guests on one host, " .
 516             "you may consider disabling this to save some context switches. " .
 517             "This is turned off by default if you use spice (-vga=qxl).",
 518     },
 519     migrate_speed => {
 520         optional => 1,
 521         type => 'integer',
 522         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 523         minimum => 0,
 524         default => 0,
 525     },
 526     migrate_downtime => {
 527         optional => 1,
 528         type => 'number',
 529         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 530         minimum => 0,
 531         default => 0.1,
 532     },
 533     cdrom => {
 534         optional => 1,
 535         type => 'string', format => 'pve-qm-ide',
 536         typetext => '<volume>',
 537         description => "This is an alias for option -ide2",
 538     },
 539     cpu => {
 540         optional => 1,
 541         description => "Emulated CPU type.",
 542         type => 'string',
 543         format => $cpu_fmt,
 544     },
 545     parent => get_standard_option('pve-snapshot-name', {
 546         optional => 1,
 547         description => "Parent snapshot name. This is used internally, and should not be modified.",
 548     }),
 549     snaptime => {
 550         optional => 1,
 551         description => "Timestamp for snapshots.",
 552         type => 'integer',
 553         minimum => 0,
 554     },
 555     vmstate => {
 556         optional => 1,
 557         type => 'string', format => 'pve-volume-id',
 558         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 559     },
 560     vmstatestorage => get_standard_option('pve-storage-id', {
 561         description => "Default storage for VM state volumes/files.",
 562         optional => 1,
 563     }),
 564     runningmachine => get_standard_option('pve-qemu-machine', {
 565         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 566     }),
 567     machine => get_standard_option('pve-qemu-machine'),
 568     arch => {
 569         description => "Virtual processor architecture. Defaults to the host.",
 570         optional => 1,
 571         type => 'string',
 572         enum => [qw(x86_64 aarch64)],
 573     },
 574     smbios1 => {
 575         description => "Specify SMBIOS type 1 fields.",
 576         type => 'string', format => 'pve-qm-smbios1',
 577         maxLength => 256,
 578         optional => 1,
 579     },
 580     protection => {
 581         optional => 1,
 582         type => 'boolean',
 583         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 584         default => 0,
 585     },
 586     bios => {
 587         optional => 1,
 588         type => 'string',
 589         enum => [ qw(seabios ovmf) ],
 590         description => "Select BIOS implementation.",
 591         default => 'seabios',
 592     },
 593     vmgenid => {
 594         type => 'string',
 595         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 596         format_description => 'UUID',
 597         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 598         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 599             " 128-bit integer value identifier to the guest OS. This allows to".
 600             " notify the guest operating system when the virtual machine is".
 601             " executed with a different configuration (e.g. snapshot execution".
 602             " or creation from a template). The guest operating system notices".
 603             " the change, and is then able to react as appropriate by marking".
 604             " its copies of distributed databases as dirty, re-initializing its".
 605             " random number generator, etc.\n".
 606             "Note that auto-creation only works when done throug API/CLI create".
 607             " or update methods, but not when manually editing the config file.",
 608         default => "1 (autogenerated)",
 609         optional => 1,
 610     },
 611 };
 612
 613 my $confdesc_cloudinit = {
 614     citype => {
 615         optional => 1,
 616         type => 'string',
 617         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 618         enum => ['configdrive2', 'nocloud'],
 619     },
 620     ciuser => {
 621         optional => 1,
 622         type => 'string',
 623         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 624     },
 625     cipassword => {
 626         optional => 1,
 627         type => 'string',
 628         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 629     },
 630     searchdomain => {
 631         optional => 1,
 632         type => 'string',
 633         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 634     },
 635     nameserver => {
 636         optional => 1,
 637         type => 'string', format => 'address-list',
 638         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 639     },
 640     sshkeys => {
 641         optional => 1,
 642         type => 'string',
 643         format => 'urlencoded',
 644         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 645     },
 646 };
 647
 648 # what about other qemu settings ?
 649 #cpu => 'string',
 650 #machine => 'string',
 651 #fda => 'file',
 652 #fdb => 'file',
 653 #mtdblock => 'file',
 654 #sd => 'file',
 655 #pflash => 'file',
 656 #snapshot => 'bool',
 657 #bootp => 'file',
 658 ##tftp => 'dir',
 659 ##smb => 'dir',
 660 #kernel => 'file',
 661 #append => 'string',
 662 #initrd => 'file',
 663 ##soundhw => 'string',
 664
 665 while (my ($k, $v) = each %$confdesc) {
 666     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 667 }
 668
 669 my $MAX_IDE_DISKS = 4;
 670 my $MAX_SCSI_DISKS = 14;
 671 my $MAX_VIRTIO_DISKS = 16;
 672 my $MAX_SATA_DISKS = 6;
 673 my $MAX_USB_DEVICES = 5;
 674 my $MAX_NETS = 32;
 675 my $MAX_UNUSED_DISKS = 256;
 676 my $MAX_HOSTPCI_DEVICES = 4;
 677 my $MAX_SERIAL_PORTS = 4;
 678 my $MAX_PARALLEL_PORTS = 3;
 679 my $MAX_NUMA = 8;
 680
 681 my $numa_fmt = {
 682     cpus => {
 683         type => "string",
 684         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 685         description => "CPUs accessing this NUMA node.",
 686         format_description => "id[-id];...",
 687     },
 688     memory => {
 689         type => "number",
 690         description => "Amount of memory this NUMA node provides.",
 691         optional => 1,
 692     },
 693     hostnodes => {
 694         type => "string",
 695         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 696         description => "Host NUMA nodes to use.",
 697         format_description => "id[-id];...",
 698         optional => 1,
 699     },
 700     policy => {
 701         type => 'string',
 702         enum => [qw(preferred bind interleave)],
 703         description => "NUMA allocation policy.",
 704         optional => 1,
 705     },
 706 };
 707 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 708 my $numadesc = {
 709     optional => 1,
 710     type => 'string', format => $numa_fmt,
 711     description => "NUMA topology.",
 712 };
 713 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 714
 715 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 716     $confdesc->{"numa$i"} = $numadesc;
 717 }
 718
 719 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 720                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 721                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 722 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 723
 724 my $net_fmt_bridge_descr = <<__EOD__;
 725 Bridge to attach the network device to. The Proxmox VE standard bridge
 726 is called 'vmbr0'.
 727
 728 If you do not specify a bridge, we create a kvm user (NATed) network
 729 device, which provides DHCP and DNS services. The following addresses
 730 are used:
 731
 732  10.0.2.2   Gateway
 733  10.0.2.3   DNS Server
 734  10.0.2.4   SMB Server
 735
 736 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 737 __EOD__
 738
 739 my $net_fmt = {
 740     macaddr => {
 741         type => 'string',
 742         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 743         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 744         format_description => "XX:XX:XX:XX:XX:XX",
 745         optional => 1,
 746     },
 747     model => {
 748         type => 'string',
 749         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 750         enum => $nic_model_list,
 751         default_key => 1,
 752     },
 753     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 754     bridge => {
 755         type => 'string',
 756         description => $net_fmt_bridge_descr,
 757         format_description => 'bridge',
 758         optional => 1,
 759     },
 760     queues => {
 761         type => 'integer',
 762         minimum => 0, maximum => 16,
 763         description => 'Number of packet queues to be used on the device.',
 764         optional => 1,
 765     },
 766     rate => {
 767         type => 'number',
 768         minimum => 0,
 769         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 770         optional => 1,
 771     },
 772     tag => {
 773         type => 'integer',
 774         minimum => 1, maximum => 4094,
 775         description => 'VLAN tag to apply to packets on this interface.',
 776         optional => 1,
 777     },
 778     trunks => {
 779         type => 'string',
 780         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 781         description => 'VLAN trunks to pass through this interface.',
 782         format_description => 'vlanid[;vlanid...]',
 783         optional => 1,
 784     },
 785     firewall => {
 786         type => 'boolean',
 787         description => 'Whether this interface should be protected by the firewall.',
 788         optional => 1,
 789     },
 790     link_down => {
 791         type => 'boolean',
 792         description => 'Whether this interface should be disconnected (like pulling the plug).',
 793         optional => 1,
 794     },
 795 };
 796
 797 my $netdesc = {
 798     optional => 1,
 799     type => 'string', format => $net_fmt,
 800     description => "Specify network devices.",
 801 };
 802
 803 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 804
 805 my $ipconfig_fmt = {
 806     ip => {
 807         type => 'string',
 808         format => 'pve-ipv4-config',
 809         format_description => 'IPv4Format/CIDR',
 810         description => 'IPv4 address in CIDR format.',
 811         optional => 1,
 812         default => 'dhcp',
 813     },
 814     gw => {
 815         type => 'string',
 816         format => 'ipv4',
 817         format_description => 'GatewayIPv4',
 818         description => 'Default gateway for IPv4 traffic.',
 819         optional => 1,
 820         requires => 'ip',
 821     },
 822     ip6 => {
 823         type => 'string',
 824         format => 'pve-ipv6-config',
 825         format_description => 'IPv6Format/CIDR',
 826         description => 'IPv6 address in CIDR format.',
 827         optional => 1,
 828         default => 'dhcp',
 829     },
 830     gw6 => {
 831         type => 'string',
 832         format => 'ipv6',
 833         format_description => 'GatewayIPv6',
 834         description => 'Default gateway for IPv6 traffic.',
 835         optional => 1,
 836         requires => 'ip6',
 837     },
 838 };
 839 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 840 my $ipconfigdesc = {
 841     optional => 1,
 842     type => 'string', format => 'pve-qm-ipconfig',
 843     description => <<'EODESCR',
 844 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 845
 846 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 847
 848 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 849 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 850
 851 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 852 EODESCR
 853 };
 854 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 855
 856 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 857     $confdesc->{"net$i"} = $netdesc;
 858     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 859 }
 860
 861 foreach my $key (keys %$confdesc_cloudinit) {
 862     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 863 }
 864
 865 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 866 sub verify_volume_id_or_qm_path {
 867     my ($volid, $noerr) = @_;
 868
 869     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 870         return $volid;
 871     }
 872
 873     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 874     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 875     if ($@) {
 876         return undef if $noerr;
 877         die $@;
 878     }
 879     return $volid;
 880 }
 881
 882 my $drivename_hash;
 883
 884 my %drivedesc_base = (
 885     volume => { alias => 'file' },
 886     file => {
 887         type => 'string',
 888         format => 'pve-volume-id-or-qm-path',
 889         default_key => 1,
 890         format_description => 'volume',
 891         description => "The drive's backing volume.",
 892     },
 893     media => {
 894         type => 'string',
 895         enum => [qw(cdrom disk)],
 896         description => "The drive's media type.",
 897         default => 'disk',
 898         optional => 1
 899     },
 900     cyls => {
 901         type => 'integer',
 902         description => "Force the drive's physical geometry to have a specific cylinder count.",
 903         optional => 1
 904     },
 905     heads => {
 906         type => 'integer',
 907         description => "Force the drive's physical geometry to have a specific head count.",
 908         optional => 1
 909     },
 910     secs => {
 911         type => 'integer',
 912         description => "Force the drive's physical geometry to have a specific sector count.",
 913         optional => 1
 914     },
 915     trans => {
 916         type => 'string',
 917         enum => [qw(none lba auto)],
 918         description => "Force disk geometry bios translation mode.",
 919         optional => 1,
 920     },
 921     snapshot => {
 922         type => 'boolean',
 923         description => "Controls qemu's snapshot mode feature."
 924             . " If activated, changes made to the disk are temporary and will"
 925             . " be discarded when the VM is shutdown.",
 926         optional => 1,
 927     },
 928     cache => {
 929         type => 'string',
 930         enum => [qw(none writethrough writeback unsafe directsync)],
 931         description => "The drive's cache mode",
 932         optional => 1,
 933     },
 934     format => get_standard_option('pve-qm-image-format'),
 935     size => {
 936         type => 'string',
 937         format => 'disk-size',
 938         format_description => 'DiskSize',
 939         description => "Disk size. This is purely informational and has no effect.",
 940         optional => 1,
 941     },
 942     backup => {
 943         type => 'boolean',
 944         description => "Whether the drive should be included when making backups.",
 945         optional => 1,
 946     },
 947     replicate => {
 948         type => 'boolean',
 949         description => 'Whether the drive should considered for replication jobs.',
 950         optional => 1,
 951         default => 1,
 952     },
 953     rerror => {
 954         type => 'string',
 955         enum => [qw(ignore report stop)],
 956         description => 'Read error action.',
 957         optional => 1,
 958     },
 959     werror => {
 960         type => 'string',
 961         enum => [qw(enospc ignore report stop)],
 962         description => 'Write error action.',
 963         optional => 1,
 964     },
 965     aio => {
 966         type => 'string',
 967         enum => [qw(native threads)],
 968         description => 'AIO type to use.',
 969         optional => 1,
 970     },
 971     discard => {
 972         type => 'string',
 973         enum => [qw(ignore on)],
 974         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 975         optional => 1,
 976     },
 977     detect_zeroes => {
 978         type => 'boolean',
 979         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 980         optional => 1,
 981     },
 982     serial => {
 983         type => 'string',
 984         format => 'urlencoded',
 985         format_description => 'serial',
 986         maxLength => 20*3, # *3 since it's %xx url enoded
 987         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 988         optional => 1,
 989     },
 990     shared => {
 991         type => 'boolean',
 992         description => 'Mark this locally-managed volume as available on all nodes',
 993         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 994         optional => 1,
 995         default => 0,
 996     }
 997 );
 998
 999 my %iothread_fmt = ( iothread => {
1000         type => 'boolean',
1001         description => "Whether to use iothreads for this drive",
1002         optional => 1,
1003 });
1004
1005 my %model_fmt = (
1006     model => {
1007         type => 'string',
1008         format => 'urlencoded',
1009         format_description => 'model',
1010         maxLength => 40*3, # *3 since it's %xx url enoded
1011         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1012         optional => 1,
1013     },
1014 );
1015
1016 my %queues_fmt = (
1017     queues => {
1018         type => 'integer',
1019         description => "Number of queues.",
1020         minimum => 2,
1021         optional => 1
1022     }
1023 );
1024
1025 my %scsiblock_fmt = (
1026     scsiblock => {
1027         type => 'boolean',
1028         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1029         optional => 1,
1030         default => 0,
1031     },
1032 );
1033
1034 my %ssd_fmt = (
1035     ssd => {
1036         type => 'boolean',
1037         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1038         optional => 1,
1039     },
1040 );
1041
1042 my $add_throttle_desc = sub {
1043     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1044     my $d = {
1045         type => $type,
1046         format_description => $unit,
1047         description => "Maximum $what in $longunit.",
1048         optional => 1,
1049     };
1050     $d->{minimum} = $minimum if defined($minimum);
1051     $drivedesc_base{$key} = $d;
1052 };
1053 # throughput: (leaky bucket)
1054 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1055 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1056 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1057 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1058 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1059 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1060 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1061 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1062 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1063
1064 # pools: (pool of IO before throttling starts taking effect)
1065 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1066 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1067 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1068 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1069 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1070 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1071
1072 # burst lengths
1073 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1074 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1075 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1076 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1077 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1078 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1079
1080 # legacy support
1081 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1082 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1083 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1084 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1085
1086 my $ide_fmt = {
1087     %drivedesc_base,
1088     %model_fmt,
1089     %ssd_fmt,
1090 };
1091 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1092
1093 my $idedesc = {
1094     optional => 1,
1095     type => 'string', format => $ide_fmt,
1096     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1097 };
1098 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1099
1100 my $scsi_fmt = {
1101     %drivedesc_base,
1102     %iothread_fmt,
1103     %queues_fmt,
1104     %scsiblock_fmt,
1105     %ssd_fmt,
1106 };
1107 my $scsidesc = {
1108     optional => 1,
1109     type => 'string', format => $scsi_fmt,
1110     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1111 };
1112 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1113
1114 my $sata_fmt = {
1115     %drivedesc_base,
1116     %ssd_fmt,
1117 };
1118 my $satadesc = {
1119     optional => 1,
1120     type => 'string', format => $sata_fmt,
1121     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1124
1125 my $virtio_fmt = {
1126     %drivedesc_base,
1127     %iothread_fmt,
1128 };
1129 my $virtiodesc = {
1130     optional => 1,
1131     type => 'string', format => $virtio_fmt,
1132     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1133 };
1134 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1135
1136 my $alldrive_fmt = {
1137     %drivedesc_base,
1138     %iothread_fmt,
1139     %model_fmt,
1140     %queues_fmt,
1141     %scsiblock_fmt,
1142     %ssd_fmt,
1143 };
1144
1145 my $efidisk_fmt = {
1146     volume => { alias => 'file' },
1147     file => {
1148         type => 'string',
1149         format => 'pve-volume-id-or-qm-path',
1150         default_key => 1,
1151         format_description => 'volume',
1152         description => "The drive's backing volume.",
1153     },
1154     format => get_standard_option('pve-qm-image-format'),
1155     size => {
1156         type => 'string',
1157         format => 'disk-size',
1158         format_description => 'DiskSize',
1159         description => "Disk size. This is purely informational and has no effect.",
1160         optional => 1,
1161     },
1162 };
1163
1164 my $efidisk_desc = {
1165     optional => 1,
1166     type => 'string', format => $efidisk_fmt,
1167     description => "Configure a Disk for storing EFI vars",
1168 };
1169
1170 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1171
1172 my $usb_fmt = {
1173     host => {
1174         default_key => 1,
1175         type => 'string', format => 'pve-qm-usb-device',
1176         format_description => 'HOSTUSBDEVICE|spice',
1177         description => <<EODESCR,
1178 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1179
1180  'bus-port(.port)*' (decimal numbers) or
1181  'vendor_id:product_id' (hexadeciaml numbers) or
1182  'spice'
1183
1184 You can use the 'lsusb -t' command to list existing usb devices.
1185
1186 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1187
1188 The value 'spice' can be used to add a usb redirection devices for spice.
1189 EODESCR
1190     },
1191     usb3 => {
1192         optional => 1,
1193         type => 'boolean',
1194         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1195         default => 0,
1196     },
1197 };
1198
1199 my $usbdesc = {
1200     optional => 1,
1201     type => 'string', format => $usb_fmt,
1202     description => "Configure an USB device (n is 0 to 4).",
1203 };
1204 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1205
1206 # NOTE: the match-groups of this regex are used in parse_hostpci
1207 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1208 my $hostpci_fmt = {
1209     host => {
1210         default_key => 1,
1211         type => 'string',
1212         pattern => qr/$PCIRE(;$PCIRE)*/,
1213         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1214         description => <<EODESCR,
1215 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1216 of PCI virtual functions of the host. HOSTPCIID syntax is:
1217
1218 'bus:dev.func' (hexadecimal numbers)
1219
1220 You can us the 'lspci' command to list existing PCI devices.
1221 EODESCR
1222     },
1223     rombar => {
1224         type => 'boolean',
1225         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1226         optional => 1,
1227         default => 1,
1228     },
1229     romfile => {
1230         type => 'string',
1231         pattern => '[^,;]+',
1232         format_description => 'string',
1233         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1234         optional => 1,
1235     },
1236     pcie => {
1237         type => 'boolean',
1238         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1239         optional => 1,
1240         default => 0,
1241     },
1242     'x-vga' => {
1243         type => 'boolean',
1244         description =>  "Enable vfio-vga device support.",
1245         optional => 1,
1246         default => 0,
1247     },
1248 };
1249 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1250
1251 my $hostpcidesc = {
1252         optional => 1,
1253         type => 'string', format => 'pve-qm-hostpci',
1254         description => "Map host PCI devices into guest.",
1255         verbose_description =>  <<EODESCR,
1256 Map host PCI devices into guest.
1257
1258 NOTE: This option allows direct access to host hardware. So it is no longer
1259 possible to migrate such machines - use with special care.
1260
1261 CAUTION: Experimental! User reported problems with this option.
1262 EODESCR
1263 };
1264 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1265
1266 my $serialdesc = {
1267         optional => 1,
1268         type => 'string',
1269         pattern => '(/dev/.+|socket)',
1270         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1271         verbose_description =>  <<EODESCR,
1272 Create a serial device inside the VM (n is 0 to 3), and pass through a
1273 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1274 host side (use 'qm terminal' to open a terminal connection).
1275
1276 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1277
1278 CAUTION: Experimental! User reported problems with this option.
1279 EODESCR
1280 };
1281
1282 my $paralleldesc= {
1283         optional => 1,
1284         type => 'string',
1285         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1286         description =>  "Map host parallel devices (n is 0 to 2).",
1287         verbose_description =>  <<EODESCR,
1288 Map host parallel devices (n is 0 to 2).
1289
1290 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1291
1292 CAUTION: Experimental! User reported problems with this option.
1293 EODESCR
1294 };
1295
1296 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1297     $confdesc->{"parallel$i"} = $paralleldesc;
1298 }
1299
1300 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1301     $confdesc->{"serial$i"} = $serialdesc;
1302 }
1303
1304 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1305     $confdesc->{"hostpci$i"} = $hostpcidesc;
1306 }
1307
1308 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1309     $drivename_hash->{"ide$i"} = 1;
1310     $confdesc->{"ide$i"} = $idedesc;
1311 }
1312
1313 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1314     $drivename_hash->{"sata$i"} = 1;
1315     $confdesc->{"sata$i"} = $satadesc;
1316 }
1317
1318 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1319     $drivename_hash->{"scsi$i"} = 1;
1320     $confdesc->{"scsi$i"} = $scsidesc ;
1321 }
1322
1323 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1324     $drivename_hash->{"virtio$i"} = 1;
1325     $confdesc->{"virtio$i"} = $virtiodesc;
1326 }
1327
1328 $drivename_hash->{efidisk0} = 1;
1329 $confdesc->{efidisk0} = $efidisk_desc;
1330
1331 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1332     $confdesc->{"usb$i"} = $usbdesc;
1333 }
1334
1335 my $unuseddesc = {
1336     optional => 1,
1337     type => 'string', format => 'pve-volume-id',
1338     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1339 };
1340
1341 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1342     $confdesc->{"unused$i"} = $unuseddesc;
1343 }
1344
1345 my $kvm_api_version = 0;
1346
1347 sub kvm_version {
1348     return $kvm_api_version if $kvm_api_version;
1349
1350     open my $fh, '<', '/dev/kvm'
1351         or return undef;
1352
1353     # 0xae00 => KVM_GET_API_VERSION
1354     $kvm_api_version = ioctl($fh, 0xae00, 0);
1355
1356     return $kvm_api_version;
1357 }
1358
1359 my $kvm_user_version;
1360
1361 sub kvm_user_version {
1362
1363     return $kvm_user_version if $kvm_user_version;
1364
1365     $kvm_user_version = 'unknown';
1366
1367     my $code = sub {
1368         my $line = shift;
1369         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1370             $kvm_user_version = $2;
1371         }
1372     };
1373
1374     eval { run_command("kvm -version", outfunc => $code); };
1375     warn $@ if $@;
1376
1377     return $kvm_user_version;
1378
1379 }
1380
1381 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1382
1383 sub valid_drive_names {
1384     # order is important - used to autoselect boot disk
1385     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1386             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1387             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1388             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1389             'efidisk0');
1390 }
1391
1392 sub is_valid_drivename {
1393     my $dev = shift;
1394
1395     return defined($drivename_hash->{$dev});
1396 }
1397
1398 sub option_exists {
1399     my $key = shift;
1400     return defined($confdesc->{$key});
1401 }
1402
1403 sub nic_models {
1404     return $nic_model_list;
1405 }
1406
1407 sub os_list_description {
1408
1409     return {
1410         other => 'Other',
1411         wxp => 'Windows XP',
1412         w2k => 'Windows 2000',
1413         w2k3 =>, 'Windows 2003',
1414         w2k8 => 'Windows 2008',
1415         wvista => 'Windows Vista',
1416         win7 => 'Windows 7',
1417         win8 => 'Windows 8/2012',
1418         win10 => 'Windows 10/2016',
1419         l24 => 'Linux 2.4',
1420         l26 => 'Linux 2.6',
1421     };
1422 }
1423
1424 my $cdrom_path;
1425
1426 sub get_cdrom_path {
1427
1428     return  $cdrom_path if $cdrom_path;
1429
1430     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1431     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1432     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1433 }
1434
1435 sub get_iso_path {
1436     my ($storecfg, $vmid, $cdrom) = @_;
1437
1438     if ($cdrom eq 'cdrom') {
1439         return get_cdrom_path();
1440     } elsif ($cdrom eq 'none') {
1441         return '';
1442     } elsif ($cdrom =~ m|^/|) {
1443         return $cdrom;
1444     } else {
1445         return PVE::Storage::path($storecfg, $cdrom);
1446     }
1447 }
1448
1449 # try to convert old style file names to volume IDs
1450 sub filename_to_volume_id {
1451     my ($vmid, $file, $media) = @_;
1452
1453      if (!($file eq 'none' || $file eq 'cdrom' ||
1454           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1455
1456         return undef if $file =~ m|/|;
1457
1458         if ($media && $media eq 'cdrom') {
1459             $file = "local:iso/$file";
1460         } else {
1461             $file = "local:$vmid/$file";
1462         }
1463     }
1464
1465     return $file;
1466 }
1467
1468 sub verify_media_type {
1469     my ($opt, $vtype, $media) = @_;
1470
1471     return if !$media;
1472
1473     my $etype;
1474     if ($media eq 'disk') {
1475         $etype = 'images';
1476     } elsif ($media eq 'cdrom') {
1477         $etype = 'iso';
1478     } else {
1479         die "internal error";
1480     }
1481
1482     return if ($vtype eq $etype);
1483
1484     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1485 }
1486
1487 sub cleanup_drive_path {
1488     my ($opt, $storecfg, $drive) = @_;
1489
1490     # try to convert filesystem paths to volume IDs
1491
1492     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1493         ($drive->{file} !~ m|^/dev/.+|) &&
1494         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1495         ($drive->{file} !~ m/^\d+$/)) {
1496         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1497         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1498         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1499         verify_media_type($opt, $vtype, $drive->{media});
1500         $drive->{file} = $volid;
1501     }
1502
1503     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1504 }
1505
1506 sub parse_hotplug_features {
1507     my ($data) = @_;
1508
1509     my $res = {};
1510
1511     return $res if $data eq '0';
1512
1513     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1514
1515     foreach my $feature (PVE::Tools::split_list($data)) {
1516         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1517             $res->{$1} = 1;
1518         } else {
1519             die "invalid hotplug feature '$feature'\n";
1520         }
1521     }
1522     return $res;
1523 }
1524
1525 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1526 sub pve_verify_hotplug_features {
1527     my ($value, $noerr) = @_;
1528
1529     return $value if parse_hotplug_features($value);
1530
1531     return undef if $noerr;
1532
1533     die "unable to parse hotplug option\n";
1534 }
1535
1536 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1537 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1538 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1539 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1540 #        [,iothread=on][,serial=serial][,model=model]
1541
1542 sub parse_drive {
1543     my ($key, $data) = @_;
1544
1545     my ($interface, $index);
1546
1547     if ($key =~ m/^([^\d]+)(\d+)$/) {
1548         $interface = $1;
1549         $index = $2;
1550     } else {
1551         return undef;
1552     }
1553
1554     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1555                                      : $confdesc->{$key}->{format};
1556     if (!$desc) {
1557         warn "invalid drive key: $key\n";
1558         return undef;
1559     }
1560     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1561     return undef if !$res;
1562     $res->{interface} = $interface;
1563     $res->{index} = $index;
1564
1565     my $error = 0;
1566     foreach my $opt (qw(bps bps_rd bps_wr)) {
1567         if (my $bps = defined(delete $res->{$opt})) {
1568             if (defined($res->{"m$opt"})) {
1569                 warn "both $opt and m$opt specified\n";
1570                 ++$error;
1571                 next;
1572             }
1573             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1574         }
1575     }
1576
1577     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1578     for my $requirement (
1579         [mbps_max => 'mbps'],
1580         [mbps_rd_max => 'mbps_rd'],
1581         [mbps_wr_max => 'mbps_wr'],
1582         [miops_max => 'miops'],
1583         [miops_rd_max => 'miops_rd'],
1584         [miops_wr_max => 'miops_wr'],
1585         [bps_max_length => 'mbps_max'],
1586         [bps_rd_max_length => 'mbps_rd_max'],
1587         [bps_wr_max_length => 'mbps_wr_max'],
1588         [iops_max_length => 'iops_max'],
1589         [iops_rd_max_length => 'iops_rd_max'],
1590         [iops_wr_max_length => 'iops_wr_max']) {
1591         my ($option, $requires) = @$requirement;
1592         if ($res->{$option} && !$res->{$requires}) {
1593             warn "$option requires $requires\n";
1594             ++$error;
1595         }
1596     }
1597
1598     return undef if $error;
1599
1600     return undef if $res->{mbps_rd} && $res->{mbps};
1601     return undef if $res->{mbps_wr} && $res->{mbps};
1602     return undef if $res->{iops_rd} && $res->{iops};
1603     return undef if $res->{iops_wr} && $res->{iops};
1604
1605     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1606         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1607         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1608         return undef if $res->{interface} eq 'virtio';
1609     }
1610
1611     if (my $size = $res->{size}) {
1612         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1613     }
1614
1615     return $res;
1616 }
1617
1618 sub print_drive {
1619     my ($vmid, $drive) = @_;
1620     my $data = { %$drive };
1621     delete $data->{$_} for qw(index interface);
1622     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1623 }
1624
1625 sub scsi_inquiry {
1626     my($fh, $noerr) = @_;
1627
1628     my $SG_IO = 0x2285;
1629     my $SG_GET_VERSION_NUM = 0x2282;
1630
1631     my $versionbuf = "\x00" x 8;
1632     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1633     if (!$ret) {
1634         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1635         return undef;
1636     }
1637     my $version = unpack("I", $versionbuf);
1638     if ($version < 30000) {
1639         die "scsi generic interface too old\n"  if !$noerr;
1640         return undef;
1641     }
1642
1643     my $buf = "\x00" x 36;
1644     my $sensebuf = "\x00" x 8;
1645     my $cmd = pack("C x3 C x1", 0x12, 36);
1646
1647     # see /usr/include/scsi/sg.h
1648     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1649
1650     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1651                       length($sensebuf), 0, length($buf), $buf,
1652                       $cmd, $sensebuf, 6000);
1653
1654     $ret = ioctl($fh, $SG_IO, $packet);
1655     if (!$ret) {
1656         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1657         return undef;
1658     }
1659
1660     my @res = unpack($sg_io_hdr_t, $packet);
1661     if ($res[17] || $res[18]) {
1662         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1663         return undef;
1664     }
1665
1666     my $res = {};
1667     (my $byte0, my $byte1, $res->{vendor},
1668      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1669
1670     $res->{removable} = $byte1 & 128 ? 1 : 0;
1671     $res->{type} = $byte0 & 31;
1672
1673     return $res;
1674 }
1675
1676 sub path_is_scsi {
1677     my ($path) = @_;
1678
1679     my $fh = IO::File->new("+<$path") || return undef;
1680     my $res = scsi_inquiry($fh, 1);
1681     close($fh);
1682
1683     return $res;
1684 }
1685
1686 sub machine_type_is_q35 {
1687     my ($conf) = @_;
1688
1689     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1690 }
1691
1692 sub print_tabletdevice_full {
1693     my ($conf) = @_;
1694
1695     my $q35 = machine_type_is_q35($conf);
1696
1697     # we use uhci for old VMs because tablet driver was buggy in older qemu
1698     my $usbbus = $q35 ? "ehci" : "uhci";
1699
1700     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1701 }
1702
1703 sub print_drivedevice_full {
1704     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1705
1706     my $device = '';
1707     my $maxdev = 0;
1708
1709     if ($drive->{interface} eq 'virtio') {
1710         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1711         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1712         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1713     } elsif ($drive->{interface} eq 'scsi') {
1714
1715         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1716         my $unit = $drive->{index} % $maxdev;
1717         my $devicetype = 'hd';
1718         my $path = '';
1719         if (drive_is_cdrom($drive)) {
1720             $devicetype = 'cd';
1721         } else {
1722             if ($drive->{file} =~ m|^/|) {
1723                 $path = $drive->{file};
1724                 if (my $info = path_is_scsi($path)) {
1725                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1726                         $devicetype = 'block';
1727                     } elsif ($info->{type} == 1) { # tape
1728                         $devicetype = 'generic';
1729                     }
1730                 }
1731             } else {
1732                  $path = PVE::Storage::path($storecfg, $drive->{file});
1733             }
1734
1735             if($path =~ m/^iscsi\:\/\//){
1736                 $devicetype = 'generic';
1737             }
1738         }
1739
1740         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1741             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1742         } else {
1743             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1744         }
1745
1746         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1747             $device .= ",rotation_rate=1";
1748         }
1749
1750     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1751         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1752         my $controller = int($drive->{index} / $maxdev);
1753         my $unit = $drive->{index} % $maxdev;
1754         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1755
1756         $device = "ide-$devicetype";
1757         if ($drive->{interface} eq 'ide') {
1758             $device .= ",bus=ide.$controller,unit=$unit";
1759         } else {
1760             $device .= ",bus=ahci$controller.$unit";
1761         }
1762         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1763
1764         if ($devicetype eq 'hd') {
1765             if (my $model = $drive->{model}) {
1766                 $model = URI::Escape::uri_unescape($model);
1767                 $device .= ",model=$model";
1768             }
1769             if ($drive->{ssd}) {
1770                 $device .= ",rotation_rate=1";
1771             }
1772         }
1773     } elsif ($drive->{interface} eq 'usb') {
1774         die "implement me";
1775         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1776     } else {
1777         die "unsupported interface type";
1778     }
1779
1780     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1781
1782     if (my $serial = $drive->{serial}) {
1783         $serial = URI::Escape::uri_unescape($serial);
1784         $device .= ",serial=$serial";
1785     }
1786
1787
1788     return $device;
1789 }
1790
1791 sub get_initiator_name {
1792     my $initiator;
1793
1794     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1795     while (defined(my $line = <$fh>)) {
1796         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1797         $initiator = $1;
1798         last;
1799     }
1800     $fh->close();
1801
1802     return $initiator;
1803 }
1804
1805 sub print_drive_full {
1806     my ($storecfg, $vmid, $drive) = @_;
1807
1808     my $path;
1809     my $volid = $drive->{file};
1810     my $format;
1811
1812     if (drive_is_cdrom($drive)) {
1813         $path = get_iso_path($storecfg, $vmid, $volid);
1814     } else {
1815         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1816         if ($storeid) {
1817             $path = PVE::Storage::path($storecfg, $volid);
1818             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1819             $format = qemu_img_format($scfg, $volname);
1820         } else {
1821             $path = $volid;
1822             $format = "raw";
1823         }
1824    }
1825
1826     my $opts = '';
1827     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1828     foreach my $o (@qemu_drive_options) {
1829         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1830     }
1831
1832     # snapshot only accepts on|off
1833     if (defined($drive->{snapshot})) {
1834         my $v = $drive->{snapshot} ? 'on' : 'off';
1835         $opts .= ",snapshot=$v";
1836     }
1837
1838     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1839         my ($dir, $qmpname) = @$type;
1840         if (my $v = $drive->{"mbps$dir"}) {
1841             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1842         }
1843         if (my $v = $drive->{"mbps${dir}_max"}) {
1844             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1845         }
1846         if (my $v = $drive->{"bps${dir}_max_length"}) {
1847             $opts .= ",throttling.bps$qmpname-max-length=$v";
1848         }
1849         if (my $v = $drive->{"iops${dir}"}) {
1850             $opts .= ",throttling.iops$qmpname=$v";
1851         }
1852         if (my $v = $drive->{"iops${dir}_max"}) {
1853             $opts .= ",throttling.iops$qmpname-max=$v";
1854         }
1855         if (my $v = $drive->{"iops${dir}_max_length"}) {
1856             $opts .= ",throttling.iops$qmpname-max-length=$v";
1857         }
1858     }
1859
1860     $opts .= ",format=$format" if $format && !$drive->{format};
1861
1862     my $cache_direct = 0;
1863
1864     if (my $cache = $drive->{cache}) {
1865         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1866     } elsif (!drive_is_cdrom($drive)) {
1867         $opts .= ",cache=none";
1868         $cache_direct = 1;
1869     }
1870
1871     # aio native works only with O_DIRECT
1872     if (!$drive->{aio}) {
1873         if($cache_direct) {
1874             $opts .= ",aio=native";
1875         } else {
1876             $opts .= ",aio=threads";
1877         }
1878     }
1879
1880     if (!drive_is_cdrom($drive)) {
1881         my $detectzeroes;
1882         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1883             $detectzeroes = 'off';
1884         } elsif ($drive->{discard}) {
1885             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1886         } else {
1887             # This used to be our default with discard not being specified:
1888             $detectzeroes = 'on';
1889         }
1890         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1891     }
1892
1893     my $pathinfo = $path ? "file=$path," : '';
1894
1895     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1896 }
1897
1898 sub print_netdevice_full {
1899     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1900
1901     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1902
1903     my $device = $net->{model};
1904     if ($net->{model} eq 'virtio') {
1905          $device = 'virtio-net-pci';
1906      };
1907
1908     my $pciaddr = print_pci_addr("$netid", $bridges);
1909     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1910     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1911         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1912         my $vectors = $net->{queues} * 2 + 2;
1913         $tmpstr .= ",vectors=$vectors,mq=on";
1914     }
1915     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1916
1917     if ($use_old_bios_files) {
1918         my $romfile;
1919         if ($device eq 'virtio-net-pci') {
1920             $romfile = 'pxe-virtio.rom';
1921         } elsif ($device eq 'e1000') {
1922             $romfile = 'pxe-e1000.rom';
1923         } elsif ($device eq 'ne2k') {
1924             $romfile = 'pxe-ne2k_pci.rom';
1925         } elsif ($device eq 'pcnet') {
1926             $romfile = 'pxe-pcnet.rom';
1927         } elsif ($device eq 'rtl8139') {
1928             $romfile = 'pxe-rtl8139.rom';
1929         }
1930         $tmpstr .= ",romfile=$romfile" if $romfile;
1931     }
1932
1933     return $tmpstr;
1934 }
1935
1936 sub print_netdev_full {
1937     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1938
1939     my $i = '';
1940     if ($netid =~ m/^net(\d+)$/) {
1941         $i = int($1);
1942     }
1943
1944     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1945
1946     my $ifname = "tap${vmid}i$i";
1947
1948     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1949     die "interface name '$ifname' is too long (max 15 character)\n"
1950         if length($ifname) >= 16;
1951
1952     my $vhostparam = '';
1953     if (is_native($arch)) {
1954         $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1955     }
1956
1957     my $vmname = $conf->{name} || "vm$vmid";
1958
1959     my $netdev = "";
1960     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1961
1962     if ($net->{bridge}) {
1963         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1964     } else {
1965         $netdev = "type=user,id=$netid,hostname=$vmname";
1966     }
1967
1968     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1969
1970     return $netdev;
1971 }
1972
1973
1974 sub print_cpu_device {
1975     my ($conf, $id) = @_;
1976
1977     my $kvm = $conf->{kvm} // 1;
1978     my $cpu = $kvm ? "kvm64" : "qemu64";
1979     if (my $cputype = $conf->{cpu}) {
1980         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1981             or die "Cannot parse cpu description: $cputype\n";
1982         $cpu = $cpuconf->{cputype};
1983     }
1984
1985     my $cores = $conf->{cores} || 1;
1986
1987     my $current_core = ($id - 1) % $cores;
1988     my $current_socket = int(($id - 1 - $current_core)/$cores);
1989
1990     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1991 }
1992
1993 my $vga_map = {
1994     'cirrus' => 'cirrus-vga',
1995     'std' => 'VGA',
1996     'vmware' => 'vmware-svga',
1997     'virtio' => 'virtio-vga',
1998 };
1999
2000 sub print_vga_device {
2001     my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
2002
2003     my $type = $vga_map->{$vga->{type}};
2004     my $vgamem_mb = $vga->{memory};
2005     if ($qxlnum) {
2006         $type = $id ? 'qxl' : 'qxl-vga';
2007     }
2008     die "no devicetype for $vga->{type}\n" if !$type;
2009
2010     my $memory = "";
2011     if ($vgamem_mb) {
2012         if ($vga->{type} eq 'virtio') {
2013             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2014             $memory = ",max_hostmem=$bytes";
2015         } elsif ($qxlnum) {
2016             # from https://www.spice-space.org/multiple-monitors.html
2017             $memory = ",vgamem_mb=$vga->{memory}";
2018             my $ram = $vgamem_mb * 4;
2019             my $vram = $vgamem_mb * 2;
2020             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2021         } else {
2022             $memory = ",vgamem_mb=$vga->{memory}";
2023         }
2024     } elsif ($qxlnum && $id) {
2025         $memory = ",ram_size=67108864,vram_size=33554432";
2026     }
2027
2028     my $q35 = machine_type_is_q35($conf);
2029     my $vgaid = "vga" . ($id // '');
2030     my $pciaddr;
2031
2032     if ($q35 && $vgaid eq 'vga') {
2033         # the first display uses pcie.0 bus on q35 machines
2034         $pciaddr = print_pcie_addr($vgaid, $bridges);
2035     } else {
2036         $pciaddr = print_pci_addr($vgaid, $bridges);
2037     }
2038
2039     return "$type,id=${vgaid}${memory}${pciaddr}";
2040 }
2041
2042 sub drive_is_cloudinit {
2043     my ($drive) = @_;
2044     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2045 }
2046
2047 sub drive_is_cdrom {
2048     my ($drive, $exclude_cloudinit) = @_;
2049
2050     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2051
2052     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2053
2054 }
2055
2056 sub parse_number_sets {
2057     my ($set) = @_;
2058     my $res = [];
2059     foreach my $part (split(/;/, $set)) {
2060         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2061             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2062             push @$res, [ $1, $2 ];
2063         } else {
2064             die "invalid range: $part\n";
2065         }
2066     }
2067     return $res;
2068 }
2069
2070 sub parse_numa {
2071     my ($data) = @_;
2072
2073     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2074     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2075     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2076     return $res;
2077 }
2078
2079 sub parse_hostpci {
2080     my ($value) = @_;
2081
2082     return undef if !$value;
2083
2084     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2085
2086     my @idlist = split(/;/, $res->{host});
2087     delete $res->{host};
2088     foreach my $id (@idlist) {
2089         if ($id =~ /^$PCIRE$/) {
2090             if (defined($2)) {
2091                 push @{$res->{pciid}}, { id => $1, function => $2 };
2092             } else {
2093                 my $pcidevices = lspci($1);
2094                 $res->{pciid} = $pcidevices->{$1};
2095             }
2096         } else {
2097             # should have been caught by parse_property_string already
2098             die "failed to parse PCI id: $id\n";
2099         }
2100     }
2101     return $res;
2102 }
2103
2104 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2105 sub parse_net {
2106     my ($data) = @_;
2107
2108     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2109     if ($@) {
2110         warn $@;
2111         return undef;
2112     }
2113     if (!defined($res->{macaddr})) {
2114         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2115         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2116     }
2117     return $res;
2118 }
2119
2120 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2121 sub parse_ipconfig {
2122     my ($data) = @_;
2123
2124     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2125     if ($@) {
2126         warn $@;
2127         return undef;
2128     }
2129
2130     if ($res->{gw} && !$res->{ip}) {
2131         warn 'gateway specified without specifying an IP address';
2132         return undef;
2133     }
2134     if ($res->{gw6} && !$res->{ip6}) {
2135         warn 'IPv6 gateway specified without specifying an IPv6 address';
2136         return undef;
2137     }
2138     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2139         warn 'gateway specified together with DHCP';
2140         return undef;
2141     }
2142     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2143         # gw6 + auto/dhcp
2144         warn "IPv6 gateway specified together with $res->{ip6} address";
2145         return undef;
2146     }
2147
2148     if (!$res->{ip} && !$res->{ip6}) {
2149         return { ip => 'dhcp', ip6 => 'dhcp' };
2150     }
2151
2152     return $res;
2153 }
2154
2155 sub print_net {
2156     my $net = shift;
2157
2158     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2159 }
2160
2161 sub add_random_macs {
2162     my ($settings) = @_;
2163
2164     foreach my $opt (keys %$settings) {
2165         next if $opt !~ m/^net(\d+)$/;
2166         my $net = parse_net($settings->{$opt});
2167         next if !$net;
2168         $settings->{$opt} = print_net($net);
2169     }
2170 }
2171
2172 sub vm_is_volid_owner {
2173     my ($storecfg, $vmid, $volid) = @_;
2174
2175     if ($volid !~  m|^/|) {
2176         my ($path, $owner);
2177         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2178         if ($owner && ($owner == $vmid)) {
2179             return 1;
2180         }
2181     }
2182
2183     return undef;
2184 }
2185
2186 sub split_flagged_list {
2187     my $text = shift || '';
2188     $text =~ s/[,;]/ /g;
2189     $text =~ s/^\s+//;
2190     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2191 }
2192
2193 sub join_flagged_list {
2194     my ($how, $lst) = @_;
2195     join $how, map { $lst->{$_} . $_ } keys %$lst;
2196 }
2197
2198 sub vmconfig_delete_pending_option {
2199     my ($conf, $key, $force) = @_;
2200
2201     delete $conf->{pending}->{$key};
2202     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2203     $pending_delete_hash->{$key} = $force ? '!' : '';
2204     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2205 }
2206
2207 sub vmconfig_undelete_pending_option {
2208     my ($conf, $key) = @_;
2209
2210     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2211     delete $pending_delete_hash->{$key};
2212
2213     if (%$pending_delete_hash) {
2214         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2215     } else {
2216         delete $conf->{pending}->{delete};
2217     }
2218 }
2219
2220 sub vmconfig_register_unused_drive {
2221     my ($storecfg, $vmid, $conf, $drive) = @_;
2222
2223     if (drive_is_cloudinit($drive)) {
2224         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2225         warn $@ if $@;
2226     } elsif (!drive_is_cdrom($drive)) {
2227         my $volid = $drive->{file};
2228         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2229             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2230         }
2231     }
2232 }
2233
2234 sub vmconfig_cleanup_pending {
2235     my ($conf) = @_;
2236
2237     # remove pending changes when nothing changed
2238     my $changes;
2239     foreach my $opt (keys %{$conf->{pending}}) {
2240         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2241             $changes = 1;
2242             delete $conf->{pending}->{$opt};
2243         }
2244     }
2245
2246     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2247     my $pending_delete_hash = {};
2248     while (my ($opt, $force) = each %$current_delete_hash) {
2249         if (defined($conf->{$opt})) {
2250             $pending_delete_hash->{$opt} = $force;
2251         } else {
2252             $changes = 1;
2253         }
2254     }
2255
2256     if (%$pending_delete_hash) {
2257         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2258     } else {
2259         delete $conf->{pending}->{delete};
2260     }
2261
2262     return $changes;
2263 }
2264
2265 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2266 my $smbios1_fmt = {
2267     uuid => {
2268         type => 'string',
2269         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2270         format_description => 'UUID',
2271         description => "Set SMBIOS1 UUID.",
2272         optional => 1,
2273     },
2274     version => {
2275         type => 'string',
2276         pattern => '\S+',
2277         format_description => 'string',
2278         description => "Set SMBIOS1 version.",
2279         optional => 1,
2280     },
2281     serial => {
2282         type => 'string',
2283         pattern => '\S+',
2284         format_description => 'string',
2285         description => "Set SMBIOS1 serial number.",
2286         optional => 1,
2287     },
2288     manufacturer => {
2289         type => 'string',
2290         pattern => '\S+',
2291         format_description => 'string',
2292         description => "Set SMBIOS1 manufacturer.",
2293         optional => 1,
2294     },
2295     product => {
2296         type => 'string',
2297         pattern => '\S+',
2298         format_description => 'string',
2299         description => "Set SMBIOS1 product ID.",
2300         optional => 1,
2301     },
2302     sku => {
2303         type => 'string',
2304         pattern => '\S+',
2305         format_description => 'string',
2306         description => "Set SMBIOS1 SKU string.",
2307         optional => 1,
2308     },
2309     family => {
2310         type => 'string',
2311         pattern => '\S+',
2312         format_description => 'string',
2313         description => "Set SMBIOS1 family string.",
2314         optional => 1,
2315     },
2316 };
2317
2318 sub parse_smbios1 {
2319     my ($data) = @_;
2320
2321     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2322     warn $@ if $@;
2323     return $res;
2324 }
2325
2326 sub print_smbios1 {
2327     my ($smbios1) = @_;
2328     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2329 }
2330
2331 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2332
2333 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2334 sub verify_bootdisk {
2335     my ($value, $noerr) = @_;
2336
2337     return $value if is_valid_drivename($value);
2338
2339     return undef if $noerr;
2340
2341     die "invalid boot disk '$value'\n";
2342 }
2343
2344 sub parse_watchdog {
2345     my ($value) = @_;
2346
2347     return undef if !$value;
2348
2349     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2350     warn $@ if $@;
2351     return $res;
2352 }
2353
2354 sub parse_guest_agent {
2355     my ($value) = @_;
2356
2357     return {} if !defined($value->{agent});
2358
2359     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2360     warn $@ if $@;
2361
2362     # if the agent is disabled ignore the other potentially set properties
2363     return {} if !$res->{enabled};
2364     return $res;
2365 }
2366
2367 sub parse_vga {
2368     my ($value) = @_;
2369
2370     return {} if !$value;
2371     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2372     warn $@ if $@;
2373     return $res;
2374 }
2375
2376 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2377 sub verify_usb_device {
2378     my ($value, $noerr) = @_;
2379
2380     return $value if parse_usb_device($value);
2381
2382     return undef if $noerr;
2383
2384     die "unable to parse usb device\n";
2385 }
2386
2387 # add JSON properties for create and set function
2388 sub json_config_properties {
2389     my $prop = shift;
2390
2391     foreach my $opt (keys %$confdesc) {
2392         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2393         $prop->{$opt} = $confdesc->{$opt};
2394     }
2395
2396     return $prop;
2397 }
2398
2399 # return copy of $confdesc_cloudinit to generate documentation
2400 sub cloudinit_config_properties {
2401
2402     return dclone($confdesc_cloudinit);
2403 }
2404
2405 sub check_type {
2406     my ($key, $value) = @_;
2407
2408     die "unknown setting '$key'\n" if !$confdesc->{$key};
2409
2410     my $type = $confdesc->{$key}->{type};
2411
2412     if (!defined($value)) {
2413         die "got undefined value\n";
2414     }
2415
2416     if ($value =~ m/[\n\r]/) {
2417         die "property contains a line feed\n";
2418     }
2419
2420     if ($type eq 'boolean') {
2421         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2422         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2423         die "type check ('boolean') failed - got '$value'\n";
2424     } elsif ($type eq 'integer') {
2425         return int($1) if $value =~ m/^(\d+)$/;
2426         die "type check ('integer') failed - got '$value'\n";
2427     } elsif ($type eq 'number') {
2428         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2429         die "type check ('number') failed - got '$value'\n";
2430     } elsif ($type eq 'string') {
2431         if (my $fmt = $confdesc->{$key}->{format}) {
2432             PVE::JSONSchema::check_format($fmt, $value);
2433             return $value;
2434         }
2435         $value =~ s/^\"(.*)\"$/$1/;
2436         return $value;
2437     } else {
2438         die "internal error"
2439     }
2440 }
2441
2442 sub check_iommu_support{
2443     #fixme : need to check IOMMU support
2444     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2445
2446     my $iommu=1;
2447     return $iommu;
2448
2449 }
2450
2451 sub touch_config {
2452     my ($vmid) = @_;
2453
2454     my $conf = PVE::QemuConfig->config_file($vmid);
2455     utime undef, undef, $conf;
2456 }
2457
2458 sub destroy_vm {
2459     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2460
2461     my $conffile = PVE::QemuConfig->config_file($vmid);
2462
2463     my $conf = PVE::QemuConfig->load_config($vmid);
2464
2465     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2466
2467     if ($conf->{template}) {
2468         # check if any base image is still used by a linked clone
2469         foreach_drive($conf, sub {
2470                 my ($ds, $drive) = @_;
2471
2472                 return if drive_is_cdrom($drive);
2473
2474                 my $volid = $drive->{file};
2475
2476                 return if !$volid || $volid =~ m|^/|;
2477
2478                 die "base volume '$volid' is still in use by linked cloned\n"
2479                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2480
2481         });
2482     }
2483
2484     # only remove disks owned by this VM
2485     foreach_drive($conf, sub {
2486         my ($ds, $drive) = @_;
2487
2488         return if drive_is_cdrom($drive, 1);
2489
2490         my $volid = $drive->{file};
2491
2492         return if !$volid || $volid =~ m|^/|;
2493
2494         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2495         return if !$path || !$owner || ($owner != $vmid);
2496
2497         eval {
2498             PVE::Storage::vdisk_free($storecfg, $volid);
2499         };
2500         warn "Could not remove disk '$volid', check manually: $@" if $@;
2501
2502     });
2503
2504     if ($keep_empty_config) {
2505         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2506     } else {
2507         unlink $conffile;
2508     }
2509
2510     # also remove unused disk
2511     eval {
2512         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2513
2514         eval {
2515             PVE::Storage::foreach_volid($dl, sub {
2516                 my ($volid, $sid, $volname, $d) = @_;
2517                 PVE::Storage::vdisk_free($storecfg, $volid);
2518             });
2519         };
2520         warn $@ if $@;
2521
2522     };
2523     warn $@ if $@;
2524 }
2525
2526 sub parse_vm_config {
2527     my ($filename, $raw) = @_;
2528
2529     return undef if !defined($raw);
2530
2531     my $res = {
2532         digest => Digest::SHA::sha1_hex($raw),
2533         snapshots => {},
2534         pending => {},
2535     };
2536
2537     $filename =~ m|/qemu-server/(\d+)\.conf$|
2538         || die "got strange filename '$filename'";
2539
2540     my $vmid = $1;
2541
2542     my $conf = $res;
2543     my $descr;
2544     my $section = '';
2545
2546     my @lines = split(/\n/, $raw);
2547     foreach my $line (@lines) {
2548         next if $line =~ m/^\s*$/;
2549
2550         if ($line =~ m/^\[PENDING\]\s*$/i) {
2551             $section = 'pending';
2552             if (defined($descr)) {
2553                 $descr =~ s/\s+$//;
2554                 $conf->{description} = $descr;
2555             }
2556             $descr = undef;
2557             $conf = $res->{$section} = {};
2558             next;
2559
2560         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2561             $section = $1;
2562             if (defined($descr)) {
2563                 $descr =~ s/\s+$//;
2564                 $conf->{description} = $descr;
2565             }
2566             $descr = undef;
2567             $conf = $res->{snapshots}->{$section} = {};
2568             next;
2569         }
2570
2571         if ($line =~ m/^\#(.*)\s*$/) {
2572             $descr = '' if !defined($descr);
2573             $descr .= PVE::Tools::decode_text($1) . "\n";
2574             next;
2575         }
2576
2577         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2578             $descr = '' if !defined($descr);
2579             $descr .= PVE::Tools::decode_text($2);
2580         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2581             $conf->{snapstate} = $1;
2582         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2583             my $key = $1;
2584             my $value = $2;
2585             $conf->{$key} = $value;
2586         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2587             my $value = $1;
2588             if ($section eq 'pending') {
2589                 $conf->{delete} = $value; # we parse this later
2590             } else {
2591                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2592             }
2593         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2594             my $key = $1;
2595             my $value = $2;
2596             eval { $value = check_type($key, $value); };
2597             if ($@) {
2598                 warn "vm $vmid - unable to parse value of '$key' - $@";
2599             } else {
2600                 $key = 'ide2' if $key eq 'cdrom';
2601                 my $fmt = $confdesc->{$key}->{format};
2602                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2603                     my $v = parse_drive($key, $value);
2604                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2605                         $v->{file} = $volid;
2606                         $value = print_drive($vmid, $v);
2607                     } else {
2608                         warn "vm $vmid - unable to parse value of '$key'\n";
2609                         next;
2610                     }
2611                 }
2612
2613                 $conf->{$key} = $value;
2614             }
2615         }
2616     }
2617
2618     if (defined($descr)) {
2619         $descr =~ s/\s+$//;
2620         $conf->{description} = $descr;
2621     }
2622     delete $res->{snapstate}; # just to be sure
2623
2624     return $res;
2625 }
2626
2627 sub write_vm_config {
2628     my ($filename, $conf) = @_;
2629
2630     delete $conf->{snapstate}; # just to be sure
2631
2632     if ($conf->{cdrom}) {
2633         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2634         $conf->{ide2} = $conf->{cdrom};
2635         delete $conf->{cdrom};
2636     }
2637
2638     # we do not use 'smp' any longer
2639     if ($conf->{sockets}) {
2640         delete $conf->{smp};
2641     } elsif ($conf->{smp}) {
2642         $conf->{sockets} = $conf->{smp};
2643         delete $conf->{cores};
2644         delete $conf->{smp};
2645     }
2646
2647     my $used_volids = {};
2648
2649     my $cleanup_config = sub {
2650         my ($cref, $pending, $snapname) = @_;
2651
2652         foreach my $key (keys %$cref) {
2653             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2654                 $key eq 'snapstate' || $key eq 'pending';
2655             my $value = $cref->{$key};
2656             if ($key eq 'delete') {
2657                 die "propertry 'delete' is only allowed in [PENDING]\n"
2658                     if !$pending;
2659                 # fixme: check syntax?
2660                 next;
2661             }
2662             eval { $value = check_type($key, $value); };
2663             die "unable to parse value of '$key' - $@" if $@;
2664
2665             $cref->{$key} = $value;
2666
2667             if (!$snapname && is_valid_drivename($key)) {
2668                 my $drive = parse_drive($key, $value);
2669                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2670             }
2671         }
2672     };
2673
2674     &$cleanup_config($conf);
2675
2676     &$cleanup_config($conf->{pending}, 1);
2677
2678     foreach my $snapname (keys %{$conf->{snapshots}}) {
2679         die "internal error" if $snapname eq 'pending';
2680         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2681     }
2682
2683     # remove 'unusedX' settings if we re-add a volume
2684     foreach my $key (keys %$conf) {
2685         my $value = $conf->{$key};
2686         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2687             delete $conf->{$key};
2688         }
2689     }
2690
2691     my $generate_raw_config = sub {
2692         my ($conf, $pending) = @_;
2693
2694         my $raw = '';
2695
2696         # add description as comment to top of file
2697         if (defined(my $descr = $conf->{description})) {
2698             if ($descr) {
2699                 foreach my $cl (split(/\n/, $descr)) {
2700                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2701                 }
2702             } else {
2703                 $raw .= "#\n" if $pending;
2704             }
2705         }
2706
2707         foreach my $key (sort keys %$conf) {
2708             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2709             $raw .= "$key: $conf->{$key}\n";
2710         }
2711         return $raw;
2712     };
2713
2714     my $raw = &$generate_raw_config($conf);
2715
2716     if (scalar(keys %{$conf->{pending}})){
2717         $raw .= "\n[PENDING]\n";
2718         $raw .= &$generate_raw_config($conf->{pending}, 1);
2719     }
2720
2721     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2722         $raw .= "\n[$snapname]\n";
2723         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2724     }
2725
2726     return $raw;
2727 }
2728
2729 sub load_defaults {
2730
2731     my $res = {};
2732
2733     # we use static defaults from our JSON schema configuration
2734     foreach my $key (keys %$confdesc) {
2735         if (defined(my $default = $confdesc->{$key}->{default})) {
2736             $res->{$key} = $default;
2737         }
2738     }
2739
2740     return $res;
2741 }
2742
2743 sub config_list {
2744     my $vmlist = PVE::Cluster::get_vmlist();
2745     my $res = {};
2746     return $res if !$vmlist || !$vmlist->{ids};
2747     my $ids = $vmlist->{ids};
2748
2749     foreach my $vmid (keys %$ids) {
2750         my $d = $ids->{$vmid};
2751         next if !$d->{node} || $d->{node} ne $nodename;
2752         next if !$d->{type} || $d->{type} ne 'qemu';
2753         $res->{$vmid}->{exists} = 1;
2754     }
2755     return $res;
2756 }
2757
2758 # test if VM uses local resources (to prevent migration)
2759 sub check_local_resources {
2760     my ($conf, $noerr) = @_;
2761
2762     my $loc_res = 0;
2763
2764     $loc_res = 1 if $conf->{hostusb}; # old syntax
2765     $loc_res = 1 if $conf->{hostpci}; # old syntax
2766
2767     foreach my $k (keys %$conf) {
2768         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2769         # sockets are safe: they will recreated be on the target side post-migrate
2770         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2771         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2772     }
2773
2774     die "VM uses local resources\n" if $loc_res && !$noerr;
2775
2776     return $loc_res;
2777 }
2778
2779 # check if used storages are available on all nodes (use by migrate)
2780 sub check_storage_availability {
2781     my ($storecfg, $conf, $node) = @_;
2782
2783     foreach_drive($conf, sub {
2784         my ($ds, $drive) = @_;
2785
2786         my $volid = $drive->{file};
2787         return if !$volid;
2788
2789         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2790         return if !$sid;
2791
2792         # check if storage is available on both nodes
2793         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2794         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2795    });
2796 }
2797
2798 # list nodes where all VM images are available (used by has_feature API)
2799 sub shared_nodes {
2800     my ($conf, $storecfg) = @_;
2801
2802     my $nodelist = PVE::Cluster::get_nodelist();
2803     my $nodehash = { map { $_ => 1 } @$nodelist };
2804     my $nodename = PVE::INotify::nodename();
2805
2806     foreach_drive($conf, sub {
2807         my ($ds, $drive) = @_;
2808
2809         my $volid = $drive->{file};
2810         return if !$volid;
2811
2812         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2813         if ($storeid) {
2814             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2815             if ($scfg->{disable}) {
2816                 $nodehash = {};
2817             } elsif (my $avail = $scfg->{nodes}) {
2818                 foreach my $node (keys %$nodehash) {
2819                     delete $nodehash->{$node} if !$avail->{$node};
2820                 }
2821             } elsif (!$scfg->{shared}) {
2822                 foreach my $node (keys %$nodehash) {
2823                     delete $nodehash->{$node} if $node ne $nodename
2824                 }
2825             }
2826         }
2827     });
2828
2829     return $nodehash
2830 }
2831
2832 sub check_cmdline {
2833     my ($pidfile, $pid) = @_;
2834
2835     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2836     if (defined($fh)) {
2837         my $line = <$fh>;
2838         $fh->close;
2839         return undef if !$line;
2840         my @param = split(/\0/, $line);
2841
2842         my $cmd = $param[0];
2843         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2844
2845         for (my $i = 0; $i < scalar (@param); $i++) {
2846             my $p = $param[$i];
2847             next if !$p;
2848             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2849                 my $p = $param[$i+1];
2850                 return 1 if $p && ($p eq $pidfile);
2851                 return undef;
2852             }
2853         }
2854     }
2855     return undef;
2856 }
2857
2858 sub check_running {
2859     my ($vmid, $nocheck, $node) = @_;
2860
2861     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2862
2863     die "unable to find configuration file for VM $vmid - no such machine\n"
2864         if !$nocheck && ! -f $filename;
2865
2866     my $pidfile = pidfile_name($vmid);
2867
2868     if (my $fd = IO::File->new("<$pidfile")) {
2869         my $st = stat($fd);
2870         my $line = <$fd>;
2871         close($fd);
2872
2873         my $mtime = $st->mtime;
2874         if ($mtime > time()) {
2875             warn "file '$filename' modified in future\n";
2876         }
2877
2878         if ($line =~ m/^(\d+)$/) {
2879             my $pid = $1;
2880             if (check_cmdline($pidfile, $pid)) {
2881                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2882                     return $pid;
2883                 }
2884             }
2885         }
2886     }
2887
2888     return undef;
2889 }
2890
2891 sub vzlist {
2892
2893     my $vzlist = config_list();
2894
2895     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2896
2897     while (defined(my $de = $fd->read)) {
2898         next if $de !~ m/^(\d+)\.pid$/;
2899         my $vmid = $1;
2900         next if !defined($vzlist->{$vmid});
2901         if (my $pid = check_running($vmid)) {
2902             $vzlist->{$vmid}->{pid} = $pid;
2903         }
2904     }
2905
2906     return $vzlist;
2907 }
2908
2909 sub disksize {
2910     my ($storecfg, $conf) = @_;
2911
2912     my $bootdisk = $conf->{bootdisk};
2913     return undef if !$bootdisk;
2914     return undef if !is_valid_drivename($bootdisk);
2915
2916     return undef if !$conf->{$bootdisk};
2917
2918     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2919     return undef if !defined($drive);
2920
2921     return undef if drive_is_cdrom($drive);
2922
2923     my $volid = $drive->{file};
2924     return undef if !$volid;
2925
2926     return $drive->{size};
2927 }
2928
2929 our $vmstatus_return_properties = {
2930     vmid => get_standard_option('pve-vmid'),
2931     status => {
2932         description => "Qemu process status.",
2933         type => 'string',
2934         enum => ['stopped', 'running'],
2935     },
2936     maxmem => {
2937         description => "Maximum memory in bytes.",
2938         type => 'integer',
2939         optional => 1,
2940         renderer => 'bytes',
2941     },
2942     maxdisk => {
2943         description => "Root disk size in bytes.",
2944         type => 'integer',
2945         optional => 1,
2946         renderer => 'bytes',
2947     },
2948     name => {
2949         description => "VM name.",
2950         type => 'string',
2951         optional => 1,
2952     },
2953     qmpstatus => {
2954         description => "Qemu QMP agent status.",
2955         type => 'string',
2956         optional => 1,
2957     },
2958     pid => {
2959         description => "PID of running qemu process.",
2960         type => 'integer',
2961         optional => 1,
2962     },
2963     uptime => {
2964         description => "Uptime.",
2965         type => 'integer',
2966         optional => 1,
2967         renderer => 'duration',
2968     },
2969     cpus => {
2970         description => "Maximum usable CPUs.",
2971         type => 'number',
2972         optional => 1,
2973     },
2974 };
2975
2976 my $last_proc_pid_stat;
2977
2978 # get VM status information
2979 # This must be fast and should not block ($full == false)
2980 # We only query KVM using QMP if $full == true (this can be slow)
2981 sub vmstatus {
2982     my ($opt_vmid, $full) = @_;
2983
2984     my $res = {};
2985
2986     my $storecfg = PVE::Storage::config();
2987
2988     my $list = vzlist();
2989     my $defaults = load_defaults();
2990
2991     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2992
2993     my $cpucount = $cpuinfo->{cpus} || 1;
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997
2998         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2999         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3000
3001         my $d = { vmid => $vmid };
3002         $d->{pid} = $list->{$vmid}->{pid};
3003
3004         # fixme: better status?
3005         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3006
3007         my $size = disksize($storecfg, $conf);
3008         if (defined($size)) {
3009             $d->{disk} = 0; # no info available
3010             $d->{maxdisk} = $size;
3011         } else {
3012             $d->{disk} = 0;
3013             $d->{maxdisk} = 0;
3014         }
3015
3016         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3017             * ($conf->{cores} || $defaults->{cores});
3018         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3019         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3020
3021         $d->{name} = $conf->{name} || "VM $vmid";
3022         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3023             : $defaults->{memory}*(1024*1024);
3024
3025         if ($conf->{balloon}) {
3026             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3027             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3028                 : $defaults->{shares};
3029         }
3030
3031         $d->{uptime} = 0;
3032         $d->{cpu} = 0;
3033         $d->{mem} = 0;
3034
3035         $d->{netout} = 0;
3036         $d->{netin} = 0;
3037
3038         $d->{diskread} = 0;
3039         $d->{diskwrite} = 0;
3040
3041         $d->{template} = PVE::QemuConfig->is_template($conf);
3042
3043         $d->{serial} = 1 if conf_has_serial($conf);
3044
3045         $res->{$vmid} = $d;
3046     }
3047
3048     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3049     foreach my $dev (keys %$netdev) {
3050         next if $dev !~ m/^tap([1-9]\d*)i/;
3051         my $vmid = $1;
3052         my $d = $res->{$vmid};
3053         next if !$d;
3054
3055         $d->{netout} += $netdev->{$dev}->{receive};
3056         $d->{netin} += $netdev->{$dev}->{transmit};
3057
3058         if ($full) {
3059             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3060             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3061         }
3062
3063     }
3064
3065     my $ctime = gettimeofday;
3066
3067     foreach my $vmid (keys %$list) {
3068
3069         my $d = $res->{$vmid};
3070         my $pid = $d->{pid};
3071         next if !$pid;
3072
3073         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3074         next if !$pstat; # not running
3075
3076         my $used = $pstat->{utime} + $pstat->{stime};
3077
3078         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3079
3080         if ($pstat->{vsize}) {
3081             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3082         }
3083
3084         my $old = $last_proc_pid_stat->{$pid};
3085         if (!$old) {
3086             $last_proc_pid_stat->{$pid} = {
3087                 time => $ctime,
3088                 used => $used,
3089                 cpu => 0,
3090             };
3091             next;
3092         }
3093
3094         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3095
3096         if ($dtime > 1000) {
3097             my $dutime = $used -  $old->{used};
3098
3099             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3100             $last_proc_pid_stat->{$pid} = {
3101                 time => $ctime,
3102                 used => $used,
3103                 cpu => $d->{cpu},
3104             };
3105         } else {
3106             $d->{cpu} = $old->{cpu};
3107         }
3108     }
3109
3110     return $res if !$full;
3111
3112     my $qmpclient = PVE::QMPClient->new();
3113
3114     my $ballooncb = sub {
3115         my ($vmid, $resp) = @_;
3116
3117         my $info = $resp->{'return'};
3118         return if !$info->{max_mem};
3119
3120         my $d = $res->{$vmid};
3121
3122         # use memory assigned to VM
3123         $d->{maxmem} = $info->{max_mem};
3124         $d->{balloon} = $info->{actual};
3125
3126         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3127             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3128             $d->{freemem} = $info->{free_mem};
3129         }
3130
3131         $d->{ballooninfo} = $info;
3132     };
3133
3134     my $blockstatscb = sub {
3135         my ($vmid, $resp) = @_;
3136         my $data = $resp->{'return'} || [];
3137         my $totalrdbytes = 0;
3138         my $totalwrbytes = 0;
3139
3140         for my $blockstat (@$data) {
3141             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3142             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3143
3144             $blockstat->{device} =~ s/drive-//;
3145             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3146         }
3147         $res->{$vmid}->{diskread} = $totalrdbytes;
3148         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3149     };
3150
3151     my $statuscb = sub {
3152         my ($vmid, $resp) = @_;
3153
3154         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3155         # this fails if ballon driver is not loaded, so this must be
3156         # the last commnand (following command are aborted if this fails).
3157         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3158
3159         my $status = 'unknown';
3160         if (!defined($status = $resp->{'return'}->{status})) {
3161             warn "unable to get VM status\n";
3162             return;
3163         }
3164
3165         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3166     };
3167
3168     foreach my $vmid (keys %$list) {
3169         next if $opt_vmid && ($vmid ne $opt_vmid);
3170         next if !$res->{$vmid}->{pid}; # not running
3171         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3172     }
3173
3174     $qmpclient->queue_execute(undef, 2);
3175
3176     foreach my $vmid (keys %$list) {
3177         next if $opt_vmid && ($vmid ne $opt_vmid);
3178         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3179     }
3180
3181     return $res;
3182 }
3183
3184 sub foreach_drive {
3185     my ($conf, $func, @param) = @_;
3186
3187     foreach my $ds (valid_drive_names()) {
3188         next if !defined($conf->{$ds});
3189
3190         my $drive = parse_drive($ds, $conf->{$ds});
3191         next if !$drive;
3192
3193         &$func($ds, $drive, @param);
3194     }
3195 }
3196
3197 sub foreach_volid {
3198     my ($conf, $func, @param) = @_;
3199
3200     my $volhash = {};
3201
3202     my $test_volid = sub {
3203         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3204
3205         return if !$volid;
3206
3207         $volhash->{$volid}->{cdrom} //= 1;
3208         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3209
3210         $volhash->{$volid}->{replicate} //= 0;
3211         $volhash->{$volid}->{replicate} = 1 if $replicate;
3212
3213         $volhash->{$volid}->{shared} //= 0;
3214         $volhash->{$volid}->{shared} = 1 if $shared;
3215
3216         $volhash->{$volid}->{referenced_in_config} //= 0;
3217         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3218
3219         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3220             if defined($snapname);
3221     };
3222
3223     foreach_drive($conf, sub {
3224         my ($ds, $drive) = @_;
3225         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3226     });
3227
3228     foreach my $snapname (keys %{$conf->{snapshots}}) {
3229         my $snap = $conf->{snapshots}->{$snapname};
3230         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3231         foreach_drive($snap, sub {
3232             my ($ds, $drive) = @_;
3233             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3234         });
3235     }
3236
3237     foreach my $volid (keys %$volhash) {
3238         &$func($volid, $volhash->{$volid}, @param);
3239     }
3240 }
3241
3242 sub conf_has_serial {
3243     my ($conf) = @_;
3244
3245     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3246         if ($conf->{"serial$i"}) {
3247             return 1;
3248         }
3249     }
3250
3251     return 0;
3252 }
3253
3254 sub vga_conf_has_spice {
3255     my ($vga) = @_;
3256
3257     my $vgaconf = parse_vga($vga);
3258     my $vgatype = $vgaconf->{type};
3259     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3260
3261     return $1 || 1;
3262 }
3263
3264 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3265 sub get_host_arch() {
3266     $host_arch = (POSIX::uname())[4] if !$host_arch;
3267     return $host_arch;
3268 }
3269
3270 sub is_native($) {
3271     my ($arch) = @_;
3272     return get_host_arch() eq $arch;
3273 }
3274
3275 my $default_machines = {
3276     x86_64 => 'pc',
3277     aarch64 => 'virt',
3278 };
3279
3280 sub get_basic_machine_info {
3281     my ($conf, $forcemachine) = @_;
3282
3283     my $arch = $conf->{arch} // get_host_arch();
3284     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3285     return ($arch, $machine);
3286 }
3287
3288 sub get_ovmf_files($) {
3289     my ($arch) = @_;
3290
3291     my $ovmf = $OVMF->{$arch}
3292         or die "no OVMF images known for architecture '$arch'\n";
3293
3294     return @$ovmf;
3295 }
3296
3297 my $Arch2Qemu = {
3298     aarch64 => '/usr/bin/qemu-system-aarch64',
3299     x86_64 => '/usr/bin/qemu-system-x86_64',
3300 };
3301 sub get_command_for_arch($) {
3302     my ($arch) = @_;
3303     return '/usr/bin/kvm' if is_native($arch);
3304
3305     my $cmd = $Arch2Qemu->{$arch}
3306         or die "don't know how to emulate architecture '$arch'\n";
3307     return $cmd;
3308 }
3309
3310 sub get_cpu_options {
3311     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3312
3313     my $cpuFlags = [];
3314     my $ostype = $conf->{ostype};
3315
3316     my $cpu = $kvm ? "kvm64" : "qemu64";
3317     if ($arch eq 'aarch64') {
3318         $cpu = 'cortex-a57';
3319     }
3320     if (my $cputype = $conf->{cpu}) {
3321         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3322             or die "Cannot parse cpu description: $cputype\n";
3323         $cpu = $cpuconf->{cputype};
3324         $kvm_off = 1 if $cpuconf->{hidden};
3325
3326         if (defined(my $flags = $cpuconf->{flags})) {
3327             push @$cpuFlags, split(";", $flags);
3328         }
3329     }
3330
3331     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3332
3333     push @$cpuFlags , '-x2apic'
3334         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3335
3336     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3337
3338     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3339
3340     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3341
3342         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3343         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3344     }
3345
3346     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3347
3348     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3349
3350     push @$cpuFlags, 'kvm=off' if $kvm_off;
3351
3352     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3353         push @$cpuFlags, "vendor=${cpu_vendor}"
3354             if $cpu_vendor ne 'default';
3355     } elsif ($arch ne 'aarch64') {
3356         die "internal error"; # should not happen
3357     }
3358
3359     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3360
3361     return ('-cpu', $cpu);
3362 }
3363
3364 sub config_to_command {
3365     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3366
3367     my $cmd = [];
3368     my $globalFlags = [];
3369     my $machineFlags = [];
3370     my $rtcFlags = [];
3371     my $devices = [];
3372     my $pciaddr = '';
3373     my $bridges = {};
3374     my $kvmver = kvm_user_version();
3375     my $vernum = 0; # unknown
3376     my $ostype = $conf->{ostype};
3377     my $winversion = windows_version($ostype);
3378     my $kvm = $conf->{kvm};
3379
3380     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3381     $kvm //= 1 if is_native($arch);
3382
3383     if ($kvm) {
3384         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3385             if !defined kvm_version();
3386     }
3387
3388     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3389         $vernum = $1*1000000+$2*1000;
3390     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3391         $vernum = $1*1000000+$2*1000+$3;
3392     }
3393
3394     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3395
3396     my $have_ovz = -f '/proc/vz/vestat';
3397
3398     my $q35 = machine_type_is_q35($conf);
3399     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3400     my $use_old_bios_files = undef;
3401     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3402
3403     my $cpuunits = defined($conf->{cpuunits}) ?
3404             $conf->{cpuunits} : $defaults->{cpuunits};
3405
3406     push @$cmd, get_command_for_arch($arch);
3407
3408     push @$cmd, '-id', $vmid;
3409
3410     my $vmname = $conf->{name} || "vm$vmid";
3411
3412     push @$cmd, '-name', $vmname;
3413
3414     my $use_virtio = 0;
3415
3416     my $qmpsocket = qmp_socket($vmid);
3417     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3418     push @$cmd, '-mon', "chardev=qmp,mode=control";
3419
3420     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3421         my $eventsocket = qmp_socket($vmid, 0, 'event');
3422         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3423         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3424     }
3425
3426     push @$cmd, '-pidfile' , pidfile_name($vmid);
3427
3428     push @$cmd, '-daemonize';
3429
3430     if ($conf->{smbios1}) {
3431         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3432     }
3433
3434     if ($conf->{vmgenid}) {
3435         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3436     }
3437
3438     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3439     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3440         die "uefi base image not found\n" if ! -f $ovmf_code;
3441
3442         my $path;
3443         my $format;
3444         if (my $efidisk = $conf->{efidisk0}) {
3445             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3446             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3447             $format = $d->{format};
3448             if ($storeid) {
3449                 $path = PVE::Storage::path($storecfg, $d->{file});
3450                 if (!defined($format)) {
3451                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3452                     $format = qemu_img_format($scfg, $volname);
3453                 }
3454             } else {
3455                 $path = $d->{file};
3456                 die "efidisk format must be specified\n"
3457                     if !defined($format);
3458             }
3459         } else {
3460             warn "no efidisk configured! Using temporary efivars disk.\n";
3461             $path = "/tmp/$vmid-ovmf.fd";
3462             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3463             $format = 'raw';
3464         }
3465
3466         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3467         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3468     }
3469
3470
3471     # add usb controllers
3472     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3473     push @$devices, @usbcontrollers if @usbcontrollers;
3474     my $vga = parse_vga($conf->{vga});
3475
3476     my $qxlnum = vga_conf_has_spice($conf->{vga});
3477     $vga->{type} = 'qxl' if $qxlnum;
3478
3479     if (!$vga->{type}) {
3480         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3481             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3482         } else {
3483             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3484         }
3485     }
3486
3487     # enable absolute mouse coordinates (needed by vnc)
3488     my $tablet;
3489     if (defined($conf->{tablet})) {
3490         $tablet = $conf->{tablet};
3491     } else {
3492         $tablet = $defaults->{tablet};
3493         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3494         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3495     }
3496
3497     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3498
3499     my $kvm_off = 0;
3500     my $gpu_passthrough;
3501
3502     # host pci devices
3503     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3504         my $d = parse_hostpci($conf->{"hostpci$i"});
3505         next if !$d;
3506
3507         my $pcie = $d->{pcie};
3508         if($pcie){
3509             die "q35 machine model is not enabled" if !$q35;
3510             $pciaddr = print_pcie_addr("hostpci$i");
3511         }else{
3512             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3513         }
3514
3515         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3516         my $romfile = $d->{romfile};
3517
3518         my $xvga = '';
3519         if ($d->{'x-vga'}) {
3520             $xvga = ',x-vga=on';
3521             $kvm_off = 1;
3522             $vga->{type} = 'none';
3523             $gpu_passthrough = 1;
3524
3525             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3526                 $xvga = "";
3527             }
3528         }
3529         my $pcidevices = $d->{pciid};
3530         my $multifunction = 1 if @$pcidevices > 1;
3531
3532         my $j=0;
3533         foreach my $pcidevice (@$pcidevices) {
3534
3535             my $id = "hostpci$i";
3536             $id .= ".$j" if $multifunction;
3537             my $addr = $pciaddr;
3538             $addr .= ".$j" if $multifunction;
3539             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3540
3541             if($j == 0){
3542                 $devicestr .= "$rombar$xvga";
3543                 $devicestr .= ",multifunction=on" if $multifunction;
3544                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3545             }
3546
3547             push @$devices, '-device', $devicestr;
3548             $j++;
3549         }
3550     }
3551
3552     # usb devices
3553     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3554     push @$devices, @usbdevices if @usbdevices;
3555     # serial devices
3556     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3557         if (my $path = $conf->{"serial$i"}) {
3558             if ($path eq 'socket') {
3559                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3560                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3561                 # On aarch64, serial0 is the UART device. Qemu only allows
3562                 # connecting UART devices via the '-serial' command line, as
3563                 # the device has a fixed slot on the hardware...
3564                 if ($arch eq 'aarch64' && $i == 0) {
3565                     push @$devices, '-serial', "chardev:serial$i";
3566                 } else {
3567                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3568                 }
3569             } else {
3570                 die "no such serial device\n" if ! -c $path;
3571                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3572                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3573             }
3574         }
3575     }
3576
3577     # parallel devices
3578     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3579         if (my $path = $conf->{"parallel$i"}) {
3580             die "no such parallel device\n" if ! -c $path;
3581             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3582             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3583             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3584         }
3585     }
3586
3587
3588     my $sockets = 1;
3589     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3590     $sockets = $conf->{sockets} if  $conf->{sockets};
3591
3592     my $cores = $conf->{cores} || 1;
3593
3594     my $maxcpus = $sockets * $cores;
3595
3596     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3597
3598     my $allowed_vcpus = $cpuinfo->{cpus};
3599
3600     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3601         if ($allowed_vcpus < $maxcpus);
3602
3603     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3604
3605         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3606         for (my $i = 2; $i <= $vcpus; $i++)  {
3607             my $cpustr = print_cpu_device($conf,$i);
3608             push @$cmd, '-device', $cpustr;
3609         }
3610
3611     } else {
3612
3613         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3614     }
3615     push @$cmd, '-nodefaults';
3616
3617     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3618
3619     my $bootindex_hash = {};
3620     my $i = 1;
3621     foreach my $o (split(//, $bootorder)) {
3622         $bootindex_hash->{$o} = $i*100;
3623         $i++;
3624     }
3625
3626     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3627
3628     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3629
3630     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3631
3632     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3633         push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
3634         my $socket = vnc_socket($vmid);
3635         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3636     } else {
3637         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3638         push @$cmd, '-nographic';
3639     }
3640
3641     # time drift fix
3642     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3643
3644     my $useLocaltime = $conf->{localtime};
3645
3646     if ($winversion >= 5) { # windows
3647         $useLocaltime = 1 if !defined($conf->{localtime});
3648
3649         # use time drift fix when acpi is enabled
3650         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3651             $tdf = 1 if !defined($conf->{tdf});
3652         }
3653     }
3654
3655     if ($winversion >= 6) {
3656         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3657         push @$cmd, '-no-hpet';
3658     }
3659
3660     push @$rtcFlags, 'driftfix=slew' if $tdf;
3661
3662     if (!$kvm) {
3663         push @$machineFlags, 'accel=tcg';
3664     }
3665
3666     if ($machine_type) {
3667         push @$machineFlags, "type=${machine_type}";
3668     }
3669
3670     if ($conf->{startdate}) {
3671         push @$rtcFlags, "base=$conf->{startdate}";
3672     } elsif ($useLocaltime) {
3673         push @$rtcFlags, 'base=localtime';
3674     }
3675
3676     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3677
3678     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3679
3680     push @$cmd, '-S' if $conf->{freeze};
3681
3682     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3683
3684     # enable sound
3685     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3686     #push @$cmd, '-soundhw', 'es1370';
3687     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3688
3689     if (parse_guest_agent($conf)->{enabled}) {
3690         my $qgasocket = qmp_socket($vmid, 1);
3691         my $pciaddr = print_pci_addr("qga0", $bridges);
3692         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3693         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3694         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3695     }
3696
3697     my $spice_port;
3698
3699     if ($qxlnum) {
3700         if ($qxlnum > 1) {
3701             if ($winversion){
3702                 for(my $i = 1; $i < $qxlnum; $i++){
3703                     push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
3704                 }
3705             } else {
3706                 # assume other OS works like Linux
3707                 my ($ram, $vram) = ("134217728", "67108864");
3708                 if ($vga->{memory}) {
3709                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3710                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3711                 }
3712                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3713                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3714             }
3715         }
3716
3717         my $pciaddr = print_pci_addr("spice", $bridges);
3718
3719         my $nodename = PVE::INotify::nodename();
3720         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3721         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3722         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3723         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3724         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3725
3726         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3727
3728         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3729         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3730         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3731     }
3732
3733     # enable balloon by default, unless explicitly disabled
3734     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3735         $pciaddr = print_pci_addr("balloon0", $bridges);
3736         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3737     }
3738
3739     if ($conf->{watchdog}) {
3740         my $wdopts = parse_watchdog($conf->{watchdog});
3741         $pciaddr = print_pci_addr("watchdog", $bridges);
3742         my $watchdog = $wdopts->{model} || 'i6300esb';
3743         push @$devices, '-device', "$watchdog$pciaddr";
3744         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3745     }
3746
3747     my $vollist = [];
3748     my $scsicontroller = {};
3749     my $ahcicontroller = {};
3750     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3751
3752     # Add iscsi initiator name if available
3753     if (my $initiator = get_initiator_name()) {
3754         push @$devices, '-iscsi', "initiator-name=$initiator";
3755     }
3756
3757     foreach_drive($conf, sub {
3758         my ($ds, $drive) = @_;
3759
3760         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3761             push @$vollist, $drive->{file};
3762         }
3763
3764         # ignore efidisk here, already added in bios/fw handling code above
3765         return if $drive->{interface} eq 'efidisk';
3766
3767         $use_virtio = 1 if $ds =~ m/^virtio/;
3768
3769         if (drive_is_cdrom ($drive)) {
3770             if ($bootindex_hash->{d}) {
3771                 $drive->{bootindex} = $bootindex_hash->{d};
3772                 $bootindex_hash->{d} += 1;
3773             }
3774         } else {
3775             if ($bootindex_hash->{c}) {
3776                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3777                 $bootindex_hash->{c} += 1;
3778             }
3779         }
3780
3781         if($drive->{interface} eq 'virtio'){
3782            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3783         }
3784
3785         if ($drive->{interface} eq 'scsi') {
3786
3787             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3788
3789             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3790             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3791
3792             my $iothread = '';
3793             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3794                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3795                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3796             } elsif ($drive->{iothread}) {
3797                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3798             }
3799
3800             my $queues = '';
3801             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3802                 $queues = ",num_queues=$drive->{queues}";
3803             }
3804
3805             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3806             $scsicontroller->{$controller}=1;
3807         }
3808
3809         if ($drive->{interface} eq 'sata') {
3810            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3811            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3812            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3813            $ahcicontroller->{$controller}=1;
3814         }
3815
3816         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3817         push @$devices, '-drive',$drive_cmd;
3818         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3819     });
3820
3821     for (my $i = 0; $i < $MAX_NETS; $i++) {
3822          next if !$conf->{"net$i"};
3823          my $d = parse_net($conf->{"net$i"});
3824          next if !$d;
3825
3826          $use_virtio = 1 if $d->{model} eq 'virtio';
3827
3828          if ($bootindex_hash->{n}) {
3829             $d->{bootindex} = $bootindex_hash->{n};
3830             $bootindex_hash->{n} += 1;
3831          }
3832
3833          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3834          push @$devices, '-netdev', $netdevfull;
3835
3836          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3837          push @$devices, '-device', $netdevicefull;
3838     }
3839
3840     if (!$q35) {
3841         # add pci bridges
3842         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3843            $bridges->{1} = 1;
3844            $bridges->{2} = 1;
3845         }
3846
3847         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3848
3849         while (my ($k, $v) = each %$bridges) {
3850             $pciaddr = print_pci_addr("pci.$k");
3851             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3852         }
3853     }
3854
3855     # add custom args
3856     if ($conf->{args}) {
3857         my $aa = PVE::Tools::split_args($conf->{args});
3858         push @$cmd, @$aa;
3859     }
3860
3861     push @$cmd, @$devices;
3862     push @$cmd, '-rtc', join(',', @$rtcFlags)
3863         if scalar(@$rtcFlags);
3864     push @$cmd, '-machine', join(',', @$machineFlags)
3865         if scalar(@$machineFlags);
3866     push @$cmd, '-global', join(',', @$globalFlags)
3867         if scalar(@$globalFlags);
3868
3869     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3870 }
3871
3872 sub vnc_socket {
3873     my ($vmid) = @_;
3874     return "${var_run_tmpdir}/$vmid.vnc";
3875 }
3876
3877 sub spice_port {
3878     my ($vmid) = @_;
3879
3880     my $res = vm_mon_cmd($vmid, 'query-spice');
3881
3882     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3883 }
3884
3885 sub qmp_socket {
3886     my ($vmid, $qga, $name) = @_;
3887     my $sockettype = $qga ? 'qga' : 'qmp';
3888     my $ext = $name ? '-'.$name : '';
3889     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3890 }
3891
3892 sub pidfile_name {
3893     my ($vmid) = @_;
3894     return "${var_run_tmpdir}/$vmid.pid";
3895 }
3896
3897 sub vm_devices_list {
3898     my ($vmid) = @_;
3899
3900     my $res = vm_mon_cmd($vmid, 'query-pci');
3901     my $devices_to_check = [];
3902     my $devices = {};
3903     foreach my $pcibus (@$res) {
3904         push @$devices_to_check, @{$pcibus->{devices}},
3905     }
3906
3907     while (@$devices_to_check) {
3908         my $to_check = [];
3909         for my $d (@$devices_to_check) {
3910             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3911             next if !$d->{'pci_bridge'};
3912
3913             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3914             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3915         }
3916         $devices_to_check = $to_check;
3917     }
3918
3919     my $resblock = vm_mon_cmd($vmid, 'query-block');
3920     foreach my $block (@$resblock) {
3921         if($block->{device} =~ m/^drive-(\S+)/){
3922                 $devices->{$1} = 1;
3923         }
3924     }
3925
3926     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3927     foreach my $mice (@$resmice) {
3928         if ($mice->{name} eq 'QEMU HID Tablet') {
3929             $devices->{tablet} = 1;
3930             last;
3931         }
3932     }
3933
3934     # for usb devices there is no query-usb
3935     # but we can iterate over the entries in
3936     # qom-list path=/machine/peripheral
3937     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3938     foreach my $per (@$resperipheral) {
3939         if ($per->{name} =~ m/^usb\d+$/) {
3940             $devices->{$per->{name}} = 1;
3941         }
3942     }
3943
3944     return $devices;
3945 }
3946
3947 sub vm_deviceplug {
3948     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3949
3950     my $q35 = machine_type_is_q35($conf);
3951
3952     my $devices_list = vm_devices_list($vmid);
3953     return 1 if defined($devices_list->{$deviceid});
3954
3955     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3956
3957     if ($deviceid eq 'tablet') {
3958
3959         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3960
3961     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3962
3963         die "usb hotplug currently not reliable\n";
3964         # since we can't reliably hot unplug all added usb devices
3965         # and usb passthrough disables live migration
3966         # we disable usb hotplugging for now
3967         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3968
3969     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3970
3971         qemu_iothread_add($vmid, $deviceid, $device);
3972
3973         qemu_driveadd($storecfg, $vmid, $device);
3974         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3975
3976         qemu_deviceadd($vmid, $devicefull);
3977         eval { qemu_deviceaddverify($vmid, $deviceid); };
3978         if (my $err = $@) {
3979             eval { qemu_drivedel($vmid, $deviceid); };
3980             warn $@ if $@;
3981             die $err;
3982         }
3983
3984     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3985
3986
3987         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3988         my $pciaddr = print_pci_addr($deviceid);
3989         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3990
3991         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3992
3993         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3994             qemu_iothread_add($vmid, $deviceid, $device);
3995             $devicefull .= ",iothread=iothread-$deviceid";
3996         }
3997
3998         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3999             $devicefull .= ",num_queues=$device->{queues}";
4000         }
4001
4002         qemu_deviceadd($vmid, $devicefull);
4003         qemu_deviceaddverify($vmid, $deviceid);
4004
4005     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4006
4007         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
4008         qemu_driveadd($storecfg, $vmid, $device);
4009
4010         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
4011         eval { qemu_deviceadd($vmid, $devicefull); };
4012         if (my $err = $@) {
4013             eval { qemu_drivedel($vmid, $deviceid); };
4014             warn $@ if $@;
4015             die $err;
4016         }
4017
4018     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4019
4020         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
4021
4022         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4023         my $use_old_bios_files = undef;
4024         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4025
4026         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
4027         qemu_deviceadd($vmid, $netdevicefull);
4028         eval { qemu_deviceaddverify($vmid, $deviceid); };
4029         if (my $err = $@) {
4030             eval { qemu_netdevdel($vmid, $deviceid); };
4031             warn $@ if $@;
4032             die $err;
4033         }
4034
4035     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4036
4037         my $bridgeid = $2;
4038         my $pciaddr = print_pci_addr($deviceid);
4039         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4040
4041         qemu_deviceadd($vmid, $devicefull);
4042         qemu_deviceaddverify($vmid, $deviceid);
4043
4044     } else {
4045         die "can't hotplug device '$deviceid'\n";
4046     }
4047
4048     return 1;
4049 }
4050
4051 # fixme: this should raise exceptions on error!
4052 sub vm_deviceunplug {
4053     my ($vmid, $conf, $deviceid) = @_;
4054
4055     my $devices_list = vm_devices_list($vmid);
4056     return 1 if !defined($devices_list->{$deviceid});
4057
4058     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4059
4060     if ($deviceid eq 'tablet') {
4061
4062         qemu_devicedel($vmid, $deviceid);
4063
4064     } elsif ($deviceid =~ m/^usb\d+$/) {
4065
4066         die "usb hotplug currently not reliable\n";
4067         # when unplugging usb devices this way,
4068         # there may be remaining usb controllers/hubs
4069         # so we disable it for now
4070         qemu_devicedel($vmid, $deviceid);
4071         qemu_devicedelverify($vmid, $deviceid);
4072
4073     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4074
4075         qemu_devicedel($vmid, $deviceid);
4076         qemu_devicedelverify($vmid, $deviceid);
4077         qemu_drivedel($vmid, $deviceid);
4078         qemu_iothread_del($conf, $vmid, $deviceid);
4079
4080     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4081
4082         qemu_devicedel($vmid, $deviceid);
4083         qemu_devicedelverify($vmid, $deviceid);
4084         qemu_iothread_del($conf, $vmid, $deviceid);
4085
4086     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4087
4088         qemu_devicedel($vmid, $deviceid);
4089         qemu_drivedel($vmid, $deviceid);
4090         qemu_deletescsihw($conf, $vmid, $deviceid);
4091
4092     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4093
4094         qemu_devicedel($vmid, $deviceid);
4095         qemu_devicedelverify($vmid, $deviceid);
4096         qemu_netdevdel($vmid, $deviceid);
4097
4098     } else {
4099         die "can't unplug device '$deviceid'\n";
4100     }
4101
4102     return 1;
4103 }
4104
4105 sub qemu_deviceadd {
4106     my ($vmid, $devicefull) = @_;
4107
4108     $devicefull = "driver=".$devicefull;
4109     my %options =  split(/[=,]/, $devicefull);
4110
4111     vm_mon_cmd($vmid, "device_add" , %options);
4112 }
4113
4114 sub qemu_devicedel {
4115     my ($vmid, $deviceid) = @_;
4116
4117     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4118 }
4119
4120 sub qemu_iothread_add {
4121     my($vmid, $deviceid, $device) = @_;
4122
4123     if ($device->{iothread}) {
4124         my $iothreads = vm_iothreads_list($vmid);
4125         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4126     }
4127 }
4128
4129 sub qemu_iothread_del {
4130     my($conf, $vmid, $deviceid) = @_;
4131
4132     my $device = parse_drive($deviceid, $conf->{$deviceid});
4133     if ($device->{iothread}) {
4134         my $iothreads = vm_iothreads_list($vmid);
4135         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4136     }
4137 }
4138
4139 sub qemu_objectadd {
4140     my($vmid, $objectid, $qomtype) = @_;
4141
4142     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4143
4144     return 1;
4145 }
4146
4147 sub qemu_objectdel {
4148     my($vmid, $objectid) = @_;
4149
4150     vm_mon_cmd($vmid, "object-del", id => $objectid);
4151
4152     return 1;
4153 }
4154
4155 sub qemu_driveadd {
4156     my ($storecfg, $vmid, $device) = @_;
4157
4158     my $drive = print_drive_full($storecfg, $vmid, $device);
4159     $drive =~ s/\\/\\\\/g;
4160     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4161
4162     # If the command succeeds qemu prints: "OK"
4163     return 1 if $ret =~ m/OK/s;
4164
4165     die "adding drive failed: $ret\n";
4166 }
4167
4168 sub qemu_drivedel {
4169     my($vmid, $deviceid) = @_;
4170
4171     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4172     $ret =~ s/^\s+//;
4173
4174     return 1 if $ret eq "";
4175
4176     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4177     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4178
4179     die "deleting drive $deviceid failed : $ret\n";
4180 }
4181
4182 sub qemu_deviceaddverify {
4183     my ($vmid, $deviceid) = @_;
4184
4185     for (my $i = 0; $i <= 5; $i++) {
4186          my $devices_list = vm_devices_list($vmid);
4187          return 1 if defined($devices_list->{$deviceid});
4188          sleep 1;
4189     }
4190
4191     die "error on hotplug device '$deviceid'\n";
4192 }
4193
4194
4195 sub qemu_devicedelverify {
4196     my ($vmid, $deviceid) = @_;
4197
4198     # need to verify that the device is correctly removed as device_del
4199     # is async and empty return is not reliable
4200
4201     for (my $i = 0; $i <= 5; $i++) {
4202          my $devices_list = vm_devices_list($vmid);
4203          return 1 if !defined($devices_list->{$deviceid});
4204          sleep 1;
4205     }
4206
4207     die "error on hot-unplugging device '$deviceid'\n";
4208 }
4209
4210 sub qemu_findorcreatescsihw {
4211     my ($storecfg, $conf, $vmid, $device) = @_;
4212
4213     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4214
4215     my $scsihwid="$controller_prefix$controller";
4216     my $devices_list = vm_devices_list($vmid);
4217
4218     if(!defined($devices_list->{$scsihwid})) {
4219         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4220     }
4221
4222     return 1;
4223 }
4224
4225 sub qemu_deletescsihw {
4226     my ($conf, $vmid, $opt) = @_;
4227
4228     my $device = parse_drive($opt, $conf->{$opt});
4229
4230     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4231         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4232         return 1;
4233     }
4234
4235     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4236
4237     my $devices_list = vm_devices_list($vmid);
4238     foreach my $opt (keys %{$devices_list}) {
4239         if (PVE::QemuServer::is_valid_drivename($opt)) {
4240             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4241             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4242                 return 1;
4243             }
4244         }
4245     }
4246
4247     my $scsihwid="scsihw$controller";
4248
4249     vm_deviceunplug($vmid, $conf, $scsihwid);
4250
4251     return 1;
4252 }
4253
4254 sub qemu_add_pci_bridge {
4255     my ($storecfg, $conf, $vmid, $device) = @_;
4256
4257     my $bridges = {};
4258
4259     my $bridgeid;
4260
4261     print_pci_addr($device, $bridges);
4262
4263     while (my ($k, $v) = each %$bridges) {
4264         $bridgeid = $k;
4265     }
4266     return 1 if !defined($bridgeid) || $bridgeid < 1;
4267
4268     my $bridge = "pci.$bridgeid";
4269     my $devices_list = vm_devices_list($vmid);
4270
4271     if (!defined($devices_list->{$bridge})) {
4272         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4273     }
4274
4275     return 1;
4276 }
4277
4278 sub qemu_set_link_status {
4279     my ($vmid, $device, $up) = @_;
4280
4281     vm_mon_cmd($vmid, "set_link", name => $device,
4282                up => $up ? JSON::true : JSON::false);
4283 }
4284
4285 sub qemu_netdevadd {
4286     my ($vmid, $conf, $device, $deviceid) = @_;
4287
4288     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4289     my %options =  split(/[=,]/, $netdev);
4290
4291     vm_mon_cmd($vmid, "netdev_add",  %options);
4292     return 1;
4293 }
4294
4295 sub qemu_netdevdel {
4296     my ($vmid, $deviceid) = @_;
4297
4298     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4299 }
4300
4301 sub qemu_usb_hotplug {
4302     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4303
4304     return if !$device;
4305
4306     # remove the old one first
4307     vm_deviceunplug($vmid, $conf, $deviceid);
4308
4309     # check if xhci controller is necessary and available
4310     if ($device->{usb3}) {
4311
4312         my $devicelist = vm_devices_list($vmid);
4313
4314         if (!$devicelist->{xhci}) {
4315             my $pciaddr = print_pci_addr("xhci");
4316             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4317         }
4318     }
4319     my $d = parse_usb_device($device->{host});
4320     $d->{usb3} = $device->{usb3};
4321
4322     # add the new one
4323     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4324 }
4325
4326 sub qemu_cpu_hotplug {
4327     my ($vmid, $conf, $vcpus) = @_;
4328
4329     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4330
4331     my $sockets = 1;
4332     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4333     $sockets = $conf->{sockets} if  $conf->{sockets};
4334     my $cores = $conf->{cores} || 1;
4335     my $maxcpus = $sockets * $cores;
4336
4337     $vcpus = $maxcpus if !$vcpus;
4338
4339     die "you can't add more vcpus than maxcpus\n"
4340         if $vcpus > $maxcpus;
4341
4342     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4343
4344     if ($vcpus < $currentvcpus) {
4345
4346         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4347
4348             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4349                 qemu_devicedel($vmid, "cpu$i");
4350                 my $retry = 0;
4351                 my $currentrunningvcpus = undef;
4352                 while (1) {
4353                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4354                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4355                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4356                     $retry++;
4357                     sleep 1;
4358                 }
4359                 #update conf after each succesfull cpu unplug
4360                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4361                 PVE::QemuConfig->write_config($vmid, $conf);
4362             }
4363         } else {
4364             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4365         }
4366
4367         return;
4368     }
4369
4370     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4371     die "vcpus in running vm does not match its configuration\n"
4372         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4373
4374     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4375
4376         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4377             my $cpustr = print_cpu_device($conf, $i);
4378             qemu_deviceadd($vmid, $cpustr);
4379
4380             my $retry = 0;
4381             my $currentrunningvcpus = undef;
4382             while (1) {
4383                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4384                 last if scalar(@{$currentrunningvcpus}) == $i;
4385                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4386                 sleep 1;
4387                 $retry++;
4388             }
4389             #update conf after each succesfull cpu hotplug
4390             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4391             PVE::QemuConfig->write_config($vmid, $conf);
4392         }
4393     } else {
4394
4395         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4396             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4397         }
4398     }
4399 }
4400
4401 sub qemu_block_set_io_throttle {
4402     my ($vmid, $deviceid,
4403         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4404         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4405         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4406         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4407
4408     return if !check_running($vmid) ;
4409
4410     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4411         bps => int($bps),
4412         bps_rd => int($bps_rd),
4413         bps_wr => int($bps_wr),
4414         iops => int($iops),
4415         iops_rd => int($iops_rd),
4416         iops_wr => int($iops_wr),
4417         bps_max => int($bps_max),
4418         bps_rd_max => int($bps_rd_max),
4419         bps_wr_max => int($bps_wr_max),
4420         iops_max => int($iops_max),
4421         iops_rd_max => int($iops_rd_max),
4422         iops_wr_max => int($iops_wr_max),
4423         bps_max_length => int($bps_max_length),
4424         bps_rd_max_length => int($bps_rd_max_length),
4425         bps_wr_max_length => int($bps_wr_max_length),
4426         iops_max_length => int($iops_max_length),
4427         iops_rd_max_length => int($iops_rd_max_length),
4428         iops_wr_max_length => int($iops_wr_max_length),
4429     );
4430
4431 }
4432
4433 # old code, only used to shutdown old VM after update
4434 sub __read_avail {
4435     my ($fh, $timeout) = @_;
4436
4437     my $sel = new IO::Select;
4438     $sel->add($fh);
4439
4440     my $res = '';
4441     my $buf;
4442
4443     my @ready;
4444     while (scalar (@ready = $sel->can_read($timeout))) {
4445         my $count;
4446         if ($count = $fh->sysread($buf, 8192)) {
4447             if ($buf =~ /^(.*)\(qemu\) $/s) {
4448                 $res .= $1;
4449                 last;
4450             } else {
4451                 $res .= $buf;
4452             }
4453         } else {
4454             if (!defined($count)) {
4455                 die "$!\n";
4456             }
4457             last;
4458         }
4459     }
4460
4461     die "monitor read timeout\n" if !scalar(@ready);
4462
4463     return $res;
4464 }
4465
4466 sub qemu_block_resize {
4467     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4468
4469     my $running = check_running($vmid);
4470
4471     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4472
4473     return if !$running;
4474
4475     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4476
4477 }
4478
4479 sub qemu_volume_snapshot {
4480     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4481
4482     my $running = check_running($vmid);
4483
4484     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4485         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4486     } else {
4487         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4488     }
4489 }
4490
4491 sub qemu_volume_snapshot_delete {
4492     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4493
4494     my $running = check_running($vmid);
4495
4496     if($running) {
4497
4498         $running = undef;
4499         my $conf = PVE::QemuConfig->load_config($vmid);
4500         foreach_drive($conf, sub {
4501             my ($ds, $drive) = @_;
4502             $running = 1 if $drive->{file} eq $volid;
4503         });
4504     }
4505
4506     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4507         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4508     } else {
4509         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4510     }
4511 }
4512
4513 sub set_migration_caps {
4514     my ($vmid) = @_;
4515
4516     my $cap_ref = [];
4517
4518     my $enabled_cap = {
4519         "auto-converge" => 1,
4520         "xbzrle" => 1,
4521         "x-rdma-pin-all" => 0,
4522         "zero-blocks" => 0,
4523         "compress" => 0
4524     };
4525
4526     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4527
4528     for my $supported_capability (@$supported_capabilities) {
4529         push @$cap_ref, {
4530             capability => $supported_capability->{capability},
4531             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4532         };
4533     }
4534
4535     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4536 }
4537
4538 my $fast_plug_option = {
4539     'lock' => 1,
4540     'name' => 1,
4541     'onboot' => 1,
4542     'shares' => 1,
4543     'startup' => 1,
4544     'description' => 1,
4545     'protection' => 1,
4546     'vmstatestorage' => 1,
4547 };
4548
4549 # hotplug changes in [PENDING]
4550 # $selection hash can be used to only apply specified options, for
4551 # example: { cores => 1 } (only apply changed 'cores')
4552 # $errors ref is used to return error messages
4553 sub vmconfig_hotplug_pending {
4554     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4555
4556     my $defaults = load_defaults();
4557
4558     # commit values which do not have any impact on running VM first
4559     # Note: those option cannot raise errors, we we do not care about
4560     # $selection and always apply them.
4561
4562     my $add_error = sub {
4563         my ($opt, $msg) = @_;
4564         $errors->{$opt} = "hotplug problem - $msg";
4565     };
4566
4567     my $changes = 0;
4568     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4569         if ($fast_plug_option->{$opt}) {
4570             $conf->{$opt} = $conf->{pending}->{$opt};
4571             delete $conf->{pending}->{$opt};
4572             $changes = 1;
4573         }
4574     }
4575
4576     if ($changes) {
4577         PVE::QemuConfig->write_config($vmid, $conf);
4578         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4579     }
4580
4581     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4582
4583     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4584     while (my ($opt, $force) = each %$pending_delete_hash) {
4585         next if $selection && !$selection->{$opt};
4586         eval {
4587             if ($opt eq 'hotplug') {
4588                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4589             } elsif ($opt eq 'tablet') {
4590                 die "skip\n" if !$hotplug_features->{usb};
4591                 if ($defaults->{tablet}) {
4592                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4593                 } else {
4594                     vm_deviceunplug($vmid, $conf, $opt);
4595                 }
4596             } elsif ($opt =~ m/^usb\d+/) {
4597                 die "skip\n";
4598                 # since we cannot reliably hot unplug usb devices
4599                 # we are disabling it
4600                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4601                 vm_deviceunplug($vmid, $conf, $opt);
4602             } elsif ($opt eq 'vcpus') {
4603                 die "skip\n" if !$hotplug_features->{cpu};
4604                 qemu_cpu_hotplug($vmid, $conf, undef);
4605             } elsif ($opt eq 'balloon') {
4606                 # enable balloon device is not hotpluggable
4607                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4608                 # here we reset the ballooning value to memory
4609                 my $balloon = $conf->{memory} || $defaults->{memory};
4610                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4611             } elsif ($fast_plug_option->{$opt}) {
4612                 # do nothing
4613             } elsif ($opt =~ m/^net(\d+)$/) {
4614                 die "skip\n" if !$hotplug_features->{network};
4615                 vm_deviceunplug($vmid, $conf, $opt);
4616             } elsif (is_valid_drivename($opt)) {
4617                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4618                 vm_deviceunplug($vmid, $conf, $opt);
4619                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4620             } elsif ($opt =~ m/^memory$/) {
4621                 die "skip\n" if !$hotplug_features->{memory};
4622                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4623             } elsif ($opt eq 'cpuunits') {
4624                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4625             } elsif ($opt eq 'cpulimit') {
4626                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4627             } else {
4628                 die "skip\n";
4629             }
4630         };
4631         if (my $err = $@) {
4632             &$add_error($opt, $err) if $err ne "skip\n";
4633         } else {
4634             # save new config if hotplug was successful
4635             delete $conf->{$opt};
4636             vmconfig_undelete_pending_option($conf, $opt);
4637             PVE::QemuConfig->write_config($vmid, $conf);
4638             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4639         }
4640     }
4641
4642     my $apply_pending_cloudinit;
4643     $apply_pending_cloudinit = sub {
4644         my ($key, $value) = @_;
4645         $apply_pending_cloudinit = sub {}; # once is enough
4646
4647         my @cloudinit_opts = keys %$confdesc_cloudinit;
4648         foreach my $opt (keys %{$conf->{pending}}) {
4649             next if !grep { $_ eq $opt } @cloudinit_opts;
4650             $conf->{$opt} = delete $conf->{pending}->{$opt};
4651         }
4652
4653         my $new_conf = { %$conf };
4654         $new_conf->{$key} = $value;
4655         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4656     };
4657
4658     foreach my $opt (keys %{$conf->{pending}}) {
4659         next if $selection && !$selection->{$opt};
4660         my $value = $conf->{pending}->{$opt};
4661         eval {
4662             if ($opt eq 'hotplug') {
4663                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4664             } elsif ($opt eq 'tablet') {
4665                 die "skip\n" if !$hotplug_features->{usb};
4666                 if ($value == 1) {
4667                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4668                 } elsif ($value == 0) {
4669                     vm_deviceunplug($vmid, $conf, $opt);
4670                 }
4671             } elsif ($opt =~ m/^usb\d+$/) {
4672                 die "skip\n";
4673                 # since we cannot reliably hot unplug usb devices
4674                 # we are disabling it
4675                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4676                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4677                 die "skip\n" if !$d;
4678                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4679             } elsif ($opt eq 'vcpus') {
4680                 die "skip\n" if !$hotplug_features->{cpu};
4681                 qemu_cpu_hotplug($vmid, $conf, $value);
4682             } elsif ($opt eq 'balloon') {
4683                 # enable/disable balloning device is not hotpluggable
4684                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4685                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4686                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4687
4688                 # allow manual ballooning if shares is set to zero
4689                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4690                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4691                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4692                 }
4693             } elsif ($opt =~ m/^net(\d+)$/) {
4694                 # some changes can be done without hotplug
4695                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4696                                     $vmid, $opt, $value);
4697             } elsif (is_valid_drivename($opt)) {
4698                 # some changes can be done without hotplug
4699                 my $drive = parse_drive($opt, $value);
4700                 if (drive_is_cloudinit($drive)) {
4701                     &$apply_pending_cloudinit($opt, $value);
4702                 }
4703                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4704                                      $vmid, $opt, $value, 1);
4705             } elsif ($opt =~ m/^memory$/) { #dimms
4706                 die "skip\n" if !$hotplug_features->{memory};
4707                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4708             } elsif ($opt eq 'cpuunits') {
4709                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4710             } elsif ($opt eq 'cpulimit') {
4711                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4712                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4713             } else {
4714                 die "skip\n";  # skip non-hot-pluggable options
4715             }
4716         };
4717         if (my $err = $@) {
4718             &$add_error($opt, $err) if $err ne "skip\n";
4719         } else {
4720             # save new config if hotplug was successful
4721             $conf->{$opt} = $value;
4722             delete $conf->{pending}->{$opt};
4723             PVE::QemuConfig->write_config($vmid, $conf);
4724             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4725         }
4726     }
4727 }
4728
4729 sub try_deallocate_drive {
4730     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4731
4732     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4733         my $volid = $drive->{file};
4734         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4735             my $sid = PVE::Storage::parse_volume_id($volid);
4736             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4737
4738             # check if the disk is really unused
4739             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4740                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4741             PVE::Storage::vdisk_free($storecfg, $volid);
4742             return 1;
4743         } else {
4744             # If vm is not owner of this disk remove from config
4745             return 1;
4746         }
4747     }
4748
4749     return undef;
4750 }
4751
4752 sub vmconfig_delete_or_detach_drive {
4753     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4754
4755     my $drive = parse_drive($opt, $conf->{$opt});
4756
4757     my $rpcenv = PVE::RPCEnvironment::get();
4758     my $authuser = $rpcenv->get_user();
4759
4760     if ($force) {
4761         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4762         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4763     } else {
4764         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4765     }
4766 }
4767
4768 sub vmconfig_apply_pending {
4769     my ($vmid, $conf, $storecfg) = @_;
4770
4771     # cold plug
4772
4773     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4774     while (my ($opt, $force) = each %$pending_delete_hash) {
4775         die "internal error" if $opt =~ m/^unused/;
4776         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4777         if (!defined($conf->{$opt})) {
4778             vmconfig_undelete_pending_option($conf, $opt);
4779             PVE::QemuConfig->write_config($vmid, $conf);
4780         } elsif (is_valid_drivename($opt)) {
4781             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4782             vmconfig_undelete_pending_option($conf, $opt);
4783             delete $conf->{$opt};
4784             PVE::QemuConfig->write_config($vmid, $conf);
4785         } else {
4786             vmconfig_undelete_pending_option($conf, $opt);
4787             delete $conf->{$opt};
4788             PVE::QemuConfig->write_config($vmid, $conf);
4789         }
4790     }
4791
4792     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4793
4794     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4795         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4796
4797         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4798             # skip if nothing changed
4799         } elsif (is_valid_drivename($opt)) {
4800             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4801                 if defined($conf->{$opt});
4802             $conf->{$opt} = $conf->{pending}->{$opt};
4803         } else {
4804             $conf->{$opt} = $conf->{pending}->{$opt};
4805         }
4806
4807         delete $conf->{pending}->{$opt};
4808         PVE::QemuConfig->write_config($vmid, $conf);
4809     }
4810 }
4811
4812 my $safe_num_ne = sub {
4813     my ($a, $b) = @_;
4814
4815     return 0 if !defined($a) && !defined($b);
4816     return 1 if !defined($a);
4817     return 1 if !defined($b);
4818
4819     return $a != $b;
4820 };
4821
4822 my $safe_string_ne = sub {
4823     my ($a, $b) = @_;
4824
4825     return 0 if !defined($a) && !defined($b);
4826     return 1 if !defined($a);
4827     return 1 if !defined($b);
4828
4829     return $a ne $b;
4830 };
4831
4832 sub vmconfig_update_net {
4833     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4834
4835     my $newnet = parse_net($value);
4836
4837     if ($conf->{$opt}) {
4838         my $oldnet = parse_net($conf->{$opt});
4839
4840         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4841             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4842             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4843             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4844
4845             # for non online change, we try to hot-unplug
4846             die "skip\n" if !$hotplug;
4847             vm_deviceunplug($vmid, $conf, $opt);
4848         } else {
4849
4850             die "internal error" if $opt !~ m/net(\d+)/;
4851             my $iface = "tap${vmid}i$1";
4852
4853             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4854                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4855                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4856                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4857                 PVE::Network::tap_unplug($iface);
4858                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4859             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4860                 # Rate can be applied on its own but any change above needs to
4861                 # include the rate in tap_plug since OVS resets everything.
4862                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4863             }
4864
4865             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4866                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4867             }
4868
4869             return 1;
4870         }
4871     }
4872
4873     if ($hotplug) {
4874         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4875     } else {
4876         die "skip\n";
4877     }
4878 }
4879
4880 sub vmconfig_update_disk {
4881     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4882
4883     # fixme: do we need force?
4884
4885     my $drive = parse_drive($opt, $value);
4886
4887     if ($conf->{$opt}) {
4888
4889         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4890
4891             my $media = $drive->{media} || 'disk';
4892             my $oldmedia = $old_drive->{media} || 'disk';
4893             die "unable to change media type\n" if $media ne $oldmedia;
4894
4895             if (!drive_is_cdrom($old_drive)) {
4896
4897                 if ($drive->{file} ne $old_drive->{file}) {
4898
4899                     die "skip\n" if !$hotplug;
4900
4901                     # unplug and register as unused
4902                     vm_deviceunplug($vmid, $conf, $opt);
4903                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4904
4905                 } else {
4906                     # update existing disk
4907
4908                     # skip non hotpluggable value
4909                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4910                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4911                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4912                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4913                         die "skip\n";
4914                     }
4915
4916                     # apply throttle
4917                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4918                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4919                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4920                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4921                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4922                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4923                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4924                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4925                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4926                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4927                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4928                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4929                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4930                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4931                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4932                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4933                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4934                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4935
4936                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4937                                                    ($drive->{mbps} || 0)*1024*1024,
4938                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4939                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4940                                                    $drive->{iops} || 0,
4941                                                    $drive->{iops_rd} || 0,
4942                                                    $drive->{iops_wr} || 0,
4943                                                    ($drive->{mbps_max} || 0)*1024*1024,
4944                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4945                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4946                                                    $drive->{iops_max} || 0,
4947                                                    $drive->{iops_rd_max} || 0,
4948                                                    $drive->{iops_wr_max} || 0,
4949                                                    $drive->{bps_max_length} || 1,
4950                                                    $drive->{bps_rd_max_length} || 1,
4951                                                    $drive->{bps_wr_max_length} || 1,
4952                                                    $drive->{iops_max_length} || 1,
4953                                                    $drive->{iops_rd_max_length} || 1,
4954                                                    $drive->{iops_wr_max_length} || 1);
4955
4956                     }
4957
4958                     return 1;
4959                 }
4960
4961             } else { # cdrom
4962
4963                 if ($drive->{file} eq 'none') {
4964                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4965                     if (drive_is_cloudinit($old_drive)) {
4966                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4967                     }
4968                 } else {
4969                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4970                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4971                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4972                 }
4973
4974                 return 1;
4975             }
4976         }
4977     }
4978
4979     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4980     # hotplug new disks
4981     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4982     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4983 }
4984
4985 sub vm_start {
4986     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4987         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4988
4989     PVE::QemuConfig->lock_config($vmid, sub {
4990         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4991
4992         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4993
4994         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4995
4996         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4997
4998         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4999             vmconfig_apply_pending($vmid, $conf, $storecfg);
5000             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5001         }
5002
5003         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5004
5005         my $defaults = load_defaults();
5006
5007         # set environment variable useful inside network script
5008         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5009
5010         my $local_volumes = {};
5011
5012         if ($targetstorage) {
5013             foreach_drive($conf, sub {
5014                 my ($ds, $drive) = @_;
5015
5016                 return if drive_is_cdrom($drive);
5017
5018                 my $volid = $drive->{file};
5019
5020                 return if !$volid;
5021
5022                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5023
5024                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5025                 return if $scfg->{shared};
5026                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5027             });
5028
5029             my $format = undef;
5030
5031             foreach my $opt (sort keys %$local_volumes) {
5032
5033                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5034                 my $drive = parse_drive($opt, $conf->{$opt});
5035
5036                 #if remote storage is specified, use default format
5037                 if ($targetstorage && $targetstorage ne "1") {
5038                     $storeid = $targetstorage;
5039                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5040                     $format = $defFormat;
5041                 } else {
5042                     #else we use same format than original
5043                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5044                     $format = qemu_img_format($scfg, $volid);
5045                 }
5046
5047                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5048                 my $newdrive = $drive;
5049                 $newdrive->{format} = $format;
5050                 $newdrive->{file} = $newvolid;
5051                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5052                 $local_volumes->{$opt} = $drivestr;
5053                 #pass drive to conf for command line
5054                 $conf->{$opt} = $drivestr;
5055             }
5056         }
5057
5058         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5059
5060         my $migrate_port = 0;
5061         my $migrate_uri;
5062         if ($statefile) {
5063             if ($statefile eq 'tcp') {
5064                 my $localip = "localhost";
5065                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5066                 my $nodename = PVE::INotify::nodename();
5067
5068                 if (!defined($migration_type)) {
5069                     if (defined($datacenterconf->{migration}->{type})) {
5070                         $migration_type = $datacenterconf->{migration}->{type};
5071                     } else {
5072                         $migration_type = 'secure';
5073                     }
5074                 }
5075
5076                 if ($migration_type eq 'insecure') {
5077                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5078                     if ($migrate_network_addr) {
5079                         $localip = $migrate_network_addr;
5080                     } else {
5081                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5082                     }
5083
5084                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5085                 }
5086
5087                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5088                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5089                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5090                 push @$cmd, '-incoming', $migrate_uri;
5091                 push @$cmd, '-S';
5092
5093             } elsif ($statefile eq 'unix') {
5094                 # should be default for secure migrations as a ssh TCP forward
5095                 # tunnel is not deterministic reliable ready and fails regurarly
5096                 # to set up in time, so use UNIX socket forwards
5097                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5098                 unlink $socket_addr;
5099
5100                 $migrate_uri = "unix:$socket_addr";
5101
5102                 push @$cmd, '-incoming', $migrate_uri;
5103                 push @$cmd, '-S';
5104
5105             } else {
5106                 push @$cmd, '-loadstate', $statefile;
5107             }
5108         } elsif ($paused) {
5109             push @$cmd, '-S';
5110         }
5111
5112         # host pci devices
5113         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5114           my $d = parse_hostpci($conf->{"hostpci$i"});
5115           next if !$d;
5116           my $pcidevices = $d->{pciid};
5117           foreach my $pcidevice (@$pcidevices) {
5118                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
5119
5120                 my $info = pci_device_info("0000:$pciid");
5121                 die "IOMMU not present\n" if !check_iommu_support();
5122                 die "no pci device info for device '$pciid'\n" if !$info;
5123                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
5124                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
5125           }
5126         }
5127
5128         PVE::Storage::activate_volumes($storecfg, $vollist);
5129
5130         if (!check_running($vmid, 1)) {
5131             eval {
5132                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5133                     outfunc => sub {}, errfunc => sub {});
5134             };
5135         }
5136
5137         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5138                                                   : $defaults->{cpuunits};
5139
5140         my $start_timeout = $conf->{hugepages} ? 300 : 30;
5141         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5142
5143         my %properties = (
5144             Slice => 'qemu.slice',
5145             KillMode => 'none',
5146             CPUShares => $cpuunits
5147         );
5148
5149         if (my $cpulimit = $conf->{cpulimit}) {
5150             $properties{CPUQuota} = int($cpulimit * 100);
5151         }
5152         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5153
5154         my $run_qemu = sub {
5155             PVE::Tools::run_fork sub {
5156                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5157                 run_command($cmd, %run_params);
5158             };
5159         };
5160
5161         if ($conf->{hugepages}) {
5162
5163             my $code = sub {
5164                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5165                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5166
5167                 PVE::QemuServer::Memory::hugepages_mount();
5168                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5169
5170                 eval { $run_qemu->() };
5171                 if (my $err = $@) {
5172                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5173                     die $err;
5174                 }
5175
5176                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5177             };
5178             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5179
5180         } else {
5181             eval { $run_qemu->() };
5182         }
5183
5184         if (my $err = $@) {
5185             # deactivate volumes if start fails
5186             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5187             die "start failed: $err";
5188         }
5189
5190         print "migration listens on $migrate_uri\n" if $migrate_uri;
5191
5192         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5193             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5194             warn $@ if $@;
5195         }
5196
5197         #start nbd server for storage migration
5198         if ($targetstorage) {
5199             my $nodename = PVE::INotify::nodename();
5200             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5201             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5202             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5203             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5204
5205             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5206
5207             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5208
5209             foreach my $opt (sort keys %$local_volumes) {
5210                 my $volid = $local_volumes->{$opt};
5211                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5212                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5213                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5214             }
5215         }
5216
5217         if ($migratedfrom) {
5218             eval {
5219                 set_migration_caps($vmid);
5220             };
5221             warn $@ if $@;
5222
5223             if ($spice_port) {
5224                 print "spice listens on port $spice_port\n";
5225                 if ($spice_ticket) {
5226                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5227                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5228                 }
5229             }
5230
5231         } else {
5232             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5233                 if !$statefile && $conf->{balloon};
5234
5235             foreach my $opt (keys %$conf) {
5236                 next if $opt !~  m/^net\d+$/;
5237                 my $nicconf = parse_net($conf->{$opt});
5238                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5239             }
5240         }
5241
5242         vm_mon_cmd_nocheck($vmid, 'qom-set',
5243                     path => "machine/peripheral/balloon0",
5244                     property => "guest-stats-polling-interval",
5245                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5246
5247     });
5248 }
5249
5250 sub vm_mon_cmd {
5251     my ($vmid, $execute, %params) = @_;
5252
5253     my $cmd = { execute => $execute, arguments => \%params };
5254     vm_qmp_command($vmid, $cmd);
5255 }
5256
5257 sub vm_mon_cmd_nocheck {
5258     my ($vmid, $execute, %params) = @_;
5259
5260     my $cmd = { execute => $execute, arguments => \%params };
5261     vm_qmp_command($vmid, $cmd, 1);
5262 }
5263
5264 sub vm_qmp_command {
5265     my ($vmid, $cmd, $nocheck) = @_;
5266
5267     my $res;
5268
5269     my $timeout;
5270     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5271         $timeout = $cmd->{arguments}->{timeout};
5272         delete $cmd->{arguments}->{timeout};
5273     }
5274
5275     eval {
5276         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5277         my $sname = qmp_socket($vmid);
5278         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5279             my $qmpclient = PVE::QMPClient->new();
5280
5281             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5282         } else {
5283             die "unable to open monitor socket\n";
5284         }
5285     };
5286     if (my $err = $@) {
5287         syslog("err", "VM $vmid qmp command failed - $err");
5288         die $err;
5289     }
5290
5291     return $res;
5292 }
5293
5294 sub vm_human_monitor_command {
5295     my ($vmid, $cmdline) = @_;
5296
5297     my $res;
5298
5299     my $cmd = {
5300         execute => 'human-monitor-command',
5301         arguments => { 'command-line' => $cmdline},
5302     };
5303
5304     return vm_qmp_command($vmid, $cmd);
5305 }
5306
5307 sub vm_commandline {
5308     my ($storecfg, $vmid) = @_;
5309
5310     my $conf = PVE::QemuConfig->load_config($vmid);
5311
5312     my $defaults = load_defaults();
5313
5314     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5315
5316     return PVE::Tools::cmd2string($cmd);
5317 }
5318
5319 sub vm_reset {
5320     my ($vmid, $skiplock) = @_;
5321
5322     PVE::QemuConfig->lock_config($vmid, sub {
5323
5324         my $conf = PVE::QemuConfig->load_config($vmid);
5325
5326         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5327
5328         vm_mon_cmd($vmid, "system_reset");
5329     });
5330 }
5331
5332 sub get_vm_volumes {
5333     my ($conf) = @_;
5334
5335     my $vollist = [];
5336     foreach_volid($conf, sub {
5337         my ($volid, $attr) = @_;
5338
5339         return if $volid =~ m|^/|;
5340
5341         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5342         return if !$sid;
5343
5344         push @$vollist, $volid;
5345     });
5346
5347     return $vollist;
5348 }
5349
5350 sub vm_stop_cleanup {
5351     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5352
5353     eval {
5354
5355         if (!$keepActive) {
5356             my $vollist = get_vm_volumes($conf);
5357             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5358         }
5359
5360         foreach my $ext (qw(mon qmp pid vnc qga)) {
5361             unlink "/var/run/qemu-server/${vmid}.$ext";
5362         }
5363
5364         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5365     };
5366     warn $@ if $@; # avoid errors - just warn
5367 }
5368
5369 # Note: use $nockeck to skip tests if VM configuration file exists.
5370 # We need that when migration VMs to other nodes (files already moved)
5371 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5372 sub vm_stop {
5373     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5374
5375     $force = 1 if !defined($force) && !$shutdown;
5376
5377     if ($migratedfrom){
5378         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5379         kill 15, $pid if $pid;
5380         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5381         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5382         return;
5383     }
5384
5385     PVE::QemuConfig->lock_config($vmid, sub {
5386
5387         my $pid = check_running($vmid, $nocheck);
5388         return if !$pid;
5389
5390         my $conf;
5391         if (!$nocheck) {
5392             $conf = PVE::QemuConfig->load_config($vmid);
5393             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5394             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5395                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5396                 $timeout = $opts->{down} if $opts->{down};
5397             }
5398         }
5399
5400         $timeout = 60 if !defined($timeout);
5401
5402         eval {
5403             if ($shutdown) {
5404                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5405                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5406                 } else {
5407                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5408                 }
5409             } else {
5410                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5411             }
5412         };
5413         my $err = $@;
5414
5415         if (!$err) {
5416             my $count = 0;
5417             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5418                 $count++;
5419                 sleep 1;
5420             }
5421
5422             if ($count >= $timeout) {
5423                 if ($force) {
5424                     warn "VM still running - terminating now with SIGTERM\n";
5425                     kill 15, $pid;
5426                 } else {
5427                     die "VM quit/powerdown failed - got timeout\n";
5428                 }
5429             } else {
5430                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5431                 return;
5432             }
5433         } else {
5434             if ($force) {
5435                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5436                 kill 15, $pid;
5437             } else {
5438                 die "VM quit/powerdown failed\n";
5439             }
5440         }
5441
5442         # wait again
5443         $timeout = 10;
5444
5445         my $count = 0;
5446         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5447             $count++;
5448             sleep 1;
5449         }
5450
5451         if ($count >= $timeout) {
5452             warn "VM still running - terminating now with SIGKILL\n";
5453             kill 9, $pid;
5454             sleep 1;
5455         }
5456
5457         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5458    });
5459 }
5460
5461 sub vm_suspend {
5462     my ($vmid, $skiplock) = @_;
5463
5464     PVE::QemuConfig->lock_config($vmid, sub {
5465
5466         my $conf = PVE::QemuConfig->load_config($vmid);
5467
5468         PVE::QemuConfig->check_lock($conf)
5469             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5470
5471         vm_mon_cmd($vmid, "stop");
5472     });
5473 }
5474
5475 sub vm_resume {
5476     my ($vmid, $skiplock, $nocheck) = @_;
5477
5478     PVE::QemuConfig->lock_config($vmid, sub {
5479
5480         my $res = vm_mon_cmd($vmid, 'query-status');
5481         my $resume_cmd = 'cont';
5482
5483         if ($res->{status} && $res->{status} eq 'suspended') {
5484             $resume_cmd = 'system_wakeup';
5485         }
5486
5487         if (!$nocheck) {
5488
5489             my $conf = PVE::QemuConfig->load_config($vmid);
5490
5491             PVE::QemuConfig->check_lock($conf)
5492                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5493
5494             vm_mon_cmd($vmid, $resume_cmd);
5495
5496         } else {
5497             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5498         }
5499     });
5500 }
5501
5502 sub vm_sendkey {
5503     my ($vmid, $skiplock, $key) = @_;
5504
5505     PVE::QemuConfig->lock_config($vmid, sub {
5506
5507         my $conf = PVE::QemuConfig->load_config($vmid);
5508
5509         # there is no qmp command, so we use the human monitor command
5510         vm_human_monitor_command($vmid, "sendkey $key");
5511     });
5512 }
5513
5514 sub vm_destroy {
5515     my ($storecfg, $vmid, $skiplock) = @_;
5516
5517     PVE::QemuConfig->lock_config($vmid, sub {
5518
5519         my $conf = PVE::QemuConfig->load_config($vmid);
5520
5521         if (!check_running($vmid)) {
5522             destroy_vm($storecfg, $vmid, undef, $skiplock);
5523         } else {
5524             die "VM $vmid is running - destroy failed\n";
5525         }
5526     });
5527 }
5528
5529 # pci helpers
5530
5531 sub file_write {
5532     my ($filename, $buf) = @_;
5533
5534     my $fh = IO::File->new($filename, "w");
5535     return undef if !$fh;
5536
5537     my $res = print $fh $buf;
5538
5539     $fh->close();
5540
5541     return $res;
5542 }
5543
5544 sub pci_device_info {
5545     my ($name) = @_;
5546
5547     my $res;
5548
5549     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5550     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5551
5552     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5553     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5554
5555     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5556     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5557
5558     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5559     return undef if !defined($product) || $product !~ s/^0x//;
5560
5561     $res = {
5562         name => $name,
5563         vendor => $vendor,
5564         product => $product,
5565         domain => $domain,
5566         bus => $bus,
5567         slot => $slot,
5568         func => $func,
5569         irq => $irq,
5570         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5571     };
5572
5573     return $res;
5574 }
5575
5576 sub pci_dev_reset {
5577     my ($dev) = @_;
5578
5579     my $name = $dev->{name};
5580
5581     my $fn = "$pcisysfs/devices/$name/reset";
5582
5583     return file_write($fn, "1");
5584 }
5585
5586 sub pci_dev_bind_to_vfio {
5587     my ($dev) = @_;
5588
5589     my $name = $dev->{name};
5590
5591     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5592
5593     if (!-d $vfio_basedir) {
5594         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5595     }
5596     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5597
5598     my $testdir = "$vfio_basedir/$name";
5599     return 1 if -d $testdir;
5600
5601     my $data = "$dev->{vendor} $dev->{product}";
5602     return undef if !file_write("$vfio_basedir/new_id", $data);
5603
5604     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5605     if (!file_write($fn, $name)) {
5606         return undef if -f $fn;
5607     }
5608
5609     $fn = "$vfio_basedir/bind";
5610     if (! -d $testdir) {
5611         return undef if !file_write($fn, $name);
5612     }
5613
5614     return -d $testdir;
5615 }
5616
5617 sub pci_dev_group_bind_to_vfio {
5618     my ($pciid) = @_;
5619
5620     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5621
5622     if (!-d $vfio_basedir) {
5623         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5624     }
5625     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5626
5627     # get IOMMU group devices
5628     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5629       my @devs = grep /^0000:/, readdir($D);
5630     closedir($D);
5631
5632     foreach my $pciid (@devs) {
5633         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5634
5635         # pci bridges, switches or root ports are not supported
5636         # they have a pci_bus subdirectory so skip them
5637         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5638
5639         my $info = pci_device_info($1);
5640         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5641     }
5642
5643     return 1;
5644 }
5645
5646 # vzdump restore implementaion
5647
5648 sub tar_archive_read_firstfile {
5649     my $archive = shift;
5650
5651     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5652
5653     # try to detect archive type first
5654     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5655         die "unable to open file '$archive'\n";
5656     my $firstfile = <$fh>;
5657     kill 15, $pid;
5658     close $fh;
5659
5660     die "ERROR: archive contaions no data\n" if !$firstfile;
5661     chomp $firstfile;
5662
5663     return $firstfile;
5664 }
5665
5666 sub tar_restore_cleanup {
5667     my ($storecfg, $statfile) = @_;
5668
5669     print STDERR "starting cleanup\n";
5670
5671     if (my $fd = IO::File->new($statfile, "r")) {
5672         while (defined(my $line = <$fd>)) {
5673             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5674                 my $volid = $2;
5675                 eval {
5676                     if ($volid =~ m|^/|) {
5677                         unlink $volid || die 'unlink failed\n';
5678                     } else {
5679                         PVE::Storage::vdisk_free($storecfg, $volid);
5680                     }
5681                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5682                 };
5683                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5684             } else {
5685                 print STDERR "unable to parse line in statfile - $line";
5686             }
5687         }
5688         $fd->close();
5689     }
5690 }
5691
5692 sub restore_archive {
5693     my ($archive, $vmid, $user, $opts) = @_;
5694
5695     my $format = $opts->{format};
5696     my $comp;
5697
5698     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5699         $format = 'tar' if !$format;
5700         $comp = 'gzip';
5701     } elsif ($archive =~ m/\.tar$/) {
5702         $format = 'tar' if !$format;
5703     } elsif ($archive =~ m/.tar.lzo$/) {
5704         $format = 'tar' if !$format;
5705         $comp = 'lzop';
5706     } elsif ($archive =~ m/\.vma$/) {
5707         $format = 'vma' if !$format;
5708     } elsif ($archive =~ m/\.vma\.gz$/) {
5709         $format = 'vma' if !$format;
5710         $comp = 'gzip';
5711     } elsif ($archive =~ m/\.vma\.lzo$/) {
5712         $format = 'vma' if !$format;
5713         $comp = 'lzop';
5714     } else {
5715         $format = 'vma' if !$format; # default
5716     }
5717
5718     # try to detect archive format
5719     if ($format eq 'tar') {
5720         return restore_tar_archive($archive, $vmid, $user, $opts);
5721     } else {
5722         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5723     }
5724 }
5725
5726 sub restore_update_config_line {
5727     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5728
5729     return if $line =~ m/^\#qmdump\#/;
5730     return if $line =~ m/^\#vzdump\#/;
5731     return if $line =~ m/^lock:/;
5732     return if $line =~ m/^unused\d+:/;
5733     return if $line =~ m/^parent:/;
5734     return if $line =~ m/^template:/; # restored VM is never a template
5735
5736     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5737     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5738         # try to convert old 1.X settings
5739         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5740         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5741             my ($model, $macaddr) = split(/\=/, $devconfig);
5742             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5743             my $net = {
5744                 model => $model,
5745                 bridge => "vmbr$ind",
5746                 macaddr => $macaddr,
5747             };
5748             my $netstr = print_net($net);
5749
5750             print $outfd "net$cookie->{netcount}: $netstr\n";
5751             $cookie->{netcount}++;
5752         }
5753     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5754         my ($id, $netstr) = ($1, $2);
5755         my $net = parse_net($netstr);
5756         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5757         $netstr = print_net($net);
5758         print $outfd "$id: $netstr\n";
5759     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5760         my $virtdev = $1;
5761         my $value = $3;
5762         my $di = parse_drive($virtdev, $value);
5763         if (defined($di->{backup}) && !$di->{backup}) {
5764             print $outfd "#$line";
5765         } elsif ($map->{$virtdev}) {
5766             delete $di->{format}; # format can change on restore
5767             $di->{file} = $map->{$virtdev};
5768             $value = print_drive($vmid, $di);
5769             print $outfd "$virtdev: $value\n";
5770         } else {
5771             print $outfd $line;
5772         }
5773     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5774         my $vmgenid = $1;
5775         if ($vmgenid ne '0') {
5776             # always generate a new vmgenid if there was a valid one setup
5777             $vmgenid = generate_uuid();
5778         }
5779         print $outfd "vmgenid: $vmgenid\n";
5780     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5781         my ($uuid, $uuid_str);
5782         UUID::generate($uuid);
5783         UUID::unparse($uuid, $uuid_str);
5784         my $smbios1 = parse_smbios1($2);
5785         $smbios1->{uuid} = $uuid_str;
5786         print $outfd $1.print_smbios1($smbios1)."\n";
5787     } else {
5788         print $outfd $line;
5789     }
5790 }
5791
5792 sub scan_volids {
5793     my ($cfg, $vmid) = @_;
5794
5795     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5796
5797     my $volid_hash = {};
5798     foreach my $storeid (keys %$info) {
5799         foreach my $item (@{$info->{$storeid}}) {
5800             next if !($item->{volid} && $item->{size});
5801             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5802             $volid_hash->{$item->{volid}} = $item;
5803         }
5804     }
5805
5806     return $volid_hash;
5807 }
5808
5809 sub is_volume_in_use {
5810     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5811
5812     my $path = PVE::Storage::path($storecfg, $volid);
5813
5814     my $scan_config = sub {
5815         my ($cref, $snapname) = @_;
5816
5817         foreach my $key (keys %$cref) {
5818             my $value = $cref->{$key};
5819             if (is_valid_drivename($key)) {
5820                 next if $skip_drive && $key eq $skip_drive;
5821                 my $drive = parse_drive($key, $value);
5822                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5823                 return 1 if $volid eq $drive->{file};
5824                 if ($drive->{file} =~ m!^/!) {
5825                     return 1 if $drive->{file} eq $path;
5826                 } else {
5827                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5828                     next if !$storeid;
5829                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5830                     next if !$scfg;
5831                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5832                 }
5833             }
5834         }
5835
5836         return 0;
5837     };
5838
5839     return 1 if &$scan_config($conf);
5840
5841     undef $skip_drive;
5842
5843     foreach my $snapname (keys %{$conf->{snapshots}}) {
5844         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5845     }
5846
5847     return 0;
5848 }
5849
5850 sub update_disksize {
5851     my ($vmid, $conf, $volid_hash) = @_;
5852
5853     my $changes;
5854     my $prefix = "VM $vmid:";
5855
5856     # used and unused disks
5857     my $referenced = {};
5858
5859     # Note: it is allowed to define multiple storages with same path (alias), so
5860     # we need to check both 'volid' and real 'path' (two different volid can point
5861     # to the same path).
5862
5863     my $referencedpath = {};
5864
5865     # update size info
5866     foreach my $opt (keys %$conf) {
5867         if (is_valid_drivename($opt)) {
5868             my $drive = parse_drive($opt, $conf->{$opt});
5869             my $volid = $drive->{file};
5870             next if !$volid;
5871
5872             $referenced->{$volid} = 1;
5873             if ($volid_hash->{$volid} &&
5874                 (my $path = $volid_hash->{$volid}->{path})) {
5875                 $referencedpath->{$path} = 1;
5876             }
5877
5878             next if drive_is_cdrom($drive);
5879             next if !$volid_hash->{$volid};
5880
5881             $drive->{size} = $volid_hash->{$volid}->{size};
5882             my $new = print_drive($vmid, $drive);
5883             if ($new ne $conf->{$opt}) {
5884                 $changes = 1;
5885                 $conf->{$opt} = $new;
5886                 print "$prefix update disk '$opt' information.\n";
5887             }
5888         }
5889     }
5890
5891     # remove 'unusedX' entry if volume is used
5892     foreach my $opt (keys %$conf) {
5893         next if $opt !~ m/^unused\d+$/;
5894         my $volid = $conf->{$opt};
5895         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5896         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5897             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5898             $changes = 1;
5899             delete $conf->{$opt};
5900         }
5901
5902         $referenced->{$volid} = 1;
5903         $referencedpath->{$path} = 1 if $path;
5904     }
5905
5906     foreach my $volid (sort keys %$volid_hash) {
5907         next if $volid =~ m/vm-$vmid-state-/;
5908         next if $referenced->{$volid};
5909         my $path = $volid_hash->{$volid}->{path};
5910         next if !$path; # just to be sure
5911         next if $referencedpath->{$path};
5912         $changes = 1;
5913         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5914         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5915         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5916     }
5917
5918     return $changes;
5919 }
5920
5921 sub rescan {
5922     my ($vmid, $nolock, $dryrun) = @_;
5923
5924     my $cfg = PVE::Storage::config();
5925
5926     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5927     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5928     foreach my $stor (keys %{$cfg->{ids}}) {
5929         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5930     }
5931
5932     print "rescan volumes...\n";
5933     my $volid_hash = scan_volids($cfg, $vmid);
5934
5935     my $updatefn =  sub {
5936         my ($vmid) = @_;
5937
5938         my $conf = PVE::QemuConfig->load_config($vmid);
5939
5940         PVE::QemuConfig->check_lock($conf);
5941
5942         my $vm_volids = {};
5943         foreach my $volid (keys %$volid_hash) {
5944             my $info = $volid_hash->{$volid};
5945             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5946         }
5947
5948         my $changes = update_disksize($vmid, $conf, $vm_volids);
5949
5950         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5951     };
5952
5953     if (defined($vmid)) {
5954         if ($nolock) {
5955             &$updatefn($vmid);
5956         } else {
5957             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5958         }
5959     } else {
5960         my $vmlist = config_list();
5961         foreach my $vmid (keys %$vmlist) {
5962             if ($nolock) {
5963                 &$updatefn($vmid);
5964             } else {
5965                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5966             }
5967         }
5968     }
5969 }
5970
5971 sub restore_vma_archive {
5972     my ($archive, $vmid, $user, $opts, $comp) = @_;
5973
5974     my $readfrom = $archive;
5975
5976     my $cfg = PVE::Storage::config();
5977     my $commands = [];
5978     my $bwlimit = $opts->{bwlimit};
5979
5980     my $dbg_cmdstring = '';
5981     my $add_pipe = sub {
5982         my ($cmd) = @_;
5983         push @$commands, $cmd;
5984         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5985         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5986         $readfrom = '-';
5987     };
5988
5989     my $input = undef;
5990     if ($archive eq '-') {
5991         $input = '<&STDIN';
5992     } else {
5993         # If we use a backup from a PVE defined storage we also consider that
5994         # storage's rate limit:
5995         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5996         if (defined($volid)) {
5997             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5998             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5999             if ($readlimit) {
6000                 print STDERR "applying read rate limit: $readlimit\n";
6001                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6002                 $add_pipe->($cstream);
6003             }
6004         }
6005     }
6006
6007     if ($comp) {
6008         my $cmd;
6009         if ($comp eq 'gzip') {
6010             $cmd = ['zcat', $readfrom];
6011         } elsif ($comp eq 'lzop') {
6012             $cmd = ['lzop', '-d', '-c', $readfrom];
6013         } else {
6014             die "unknown compression method '$comp'\n";
6015         }
6016         $add_pipe->($cmd);
6017     }
6018
6019     my $tmpdir = "/var/tmp/vzdumptmp$$";
6020     rmtree $tmpdir;
6021
6022     # disable interrupts (always do cleanups)
6023     local $SIG{INT} =
6024         local $SIG{TERM} =
6025         local $SIG{QUIT} =
6026         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6027
6028     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6029     POSIX::mkfifo($mapfifo, 0600);
6030     my $fifofh;
6031
6032     my $openfifo = sub {
6033         open($fifofh, '>', $mapfifo) || die $!;
6034     };
6035
6036     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6037
6038     my $oldtimeout;
6039     my $timeout = 5;
6040
6041     my $devinfo = {};
6042
6043     my $rpcenv = PVE::RPCEnvironment::get();
6044
6045     my $conffile = PVE::QemuConfig->config_file($vmid);
6046     my $tmpfn = "$conffile.$$.tmp";
6047
6048     # Note: $oldconf is undef if VM does not exists
6049     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6050     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6051
6052     my %storage_limits;
6053
6054     my $print_devmap = sub {
6055         my $virtdev_hash = {};
6056
6057         my $cfgfn = "$tmpdir/qemu-server.conf";
6058
6059         # we can read the config - that is already extracted
6060         my $fh = IO::File->new($cfgfn, "r") ||
6061             "unable to read qemu-server.conf - $!\n";
6062
6063         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6064         if (-f $fwcfgfn) {
6065             my $pve_firewall_dir = '/etc/pve/firewall';
6066             mkdir $pve_firewall_dir; # make sure the dir exists
6067             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6068         }
6069
6070         while (defined(my $line = <$fh>)) {
6071             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6072                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6073                 die "archive does not contain data for drive '$virtdev'\n"
6074                     if !$devinfo->{$devname};
6075                 if (defined($opts->{storage})) {
6076                     $storeid = $opts->{storage} || 'local';
6077                 } elsif (!$storeid) {
6078                     $storeid = 'local';
6079                 }
6080                 $format = 'raw' if !$format;
6081                 $devinfo->{$devname}->{devname} = $devname;
6082                 $devinfo->{$devname}->{virtdev} = $virtdev;
6083                 $devinfo->{$devname}->{format} = $format;
6084                 $devinfo->{$devname}->{storeid} = $storeid;
6085
6086                 # check permission on storage
6087                 my $pool = $opts->{pool}; # todo: do we need that?
6088                 if ($user ne 'root@pam') {
6089                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6090                 }
6091
6092                 $storage_limits{$storeid} = $bwlimit;
6093
6094                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6095             }
6096         }
6097
6098         foreach my $key (keys %storage_limits) {
6099             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6100             next if !$limit;
6101             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6102             $storage_limits{$key} = $limit * 1024;
6103         }
6104
6105         foreach my $devname (keys %$devinfo) {
6106             die "found no device mapping information for device '$devname'\n"
6107                 if !$devinfo->{$devname}->{virtdev};
6108         }
6109
6110         # create empty/temp config
6111         if ($oldconf) {
6112             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6113             foreach_drive($oldconf, sub {
6114                 my ($ds, $drive) = @_;
6115
6116                 return if drive_is_cdrom($drive);
6117
6118                 my $volid = $drive->{file};
6119
6120                 return if !$volid || $volid =~ m|^/|;
6121
6122                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6123                 return if !$path || !$owner || ($owner != $vmid);
6124
6125                 # Note: only delete disk we want to restore
6126                 # other volumes will become unused
6127                 if ($virtdev_hash->{$ds}) {
6128                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6129                     if (my $err = $@) {
6130                         warn $err;
6131                     }
6132                 }
6133             });
6134
6135             # delete vmstate files
6136             # since after the restore we have no snapshots anymore
6137             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6138                 my $snap = $oldconf->{snapshots}->{$snapname};
6139                 if ($snap->{vmstate}) {
6140                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6141                     if (my $err = $@) {
6142                         warn $err;
6143                     }
6144                 }
6145             }
6146         }
6147
6148         my $map = {};
6149         foreach my $virtdev (sort keys %$virtdev_hash) {
6150             my $d = $virtdev_hash->{$virtdev};
6151             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6152             my $storeid = $d->{storeid};
6153             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6154
6155             my $map_opts = '';
6156             if (my $limit = $storage_limits{$storeid}) {
6157                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6158             }
6159
6160             # test if requested format is supported
6161             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6162             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6163             $d->{format} = $defFormat if !$supported;
6164
6165             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6166                                                   $d->{format}, undef, $alloc_size);
6167             print STDERR "new volume ID is '$volid'\n";
6168             $d->{volid} = $volid;
6169             my $path = PVE::Storage::path($cfg, $volid);
6170
6171             PVE::Storage::activate_volumes($cfg,[$volid]);
6172
6173             my $write_zeros = 1;
6174             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6175                 $write_zeros = 0;
6176             }
6177
6178             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6179
6180             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6181             $map->{$virtdev} = $volid;
6182         }
6183
6184         $fh->seek(0, 0) || die "seek failed - $!\n";
6185
6186         my $outfd = new IO::File ($tmpfn, "w") ||
6187             die "unable to write config for VM $vmid\n";
6188
6189         my $cookie = { netcount => 0 };
6190         while (defined(my $line = <$fh>)) {
6191             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6192         }
6193
6194         $fh->close();
6195         $outfd->close();
6196     };
6197
6198     eval {
6199         # enable interrupts
6200         local $SIG{INT} =
6201             local $SIG{TERM} =
6202             local $SIG{QUIT} =
6203             local $SIG{HUP} =
6204             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6205         local $SIG{ALRM} = sub { die "got timeout\n"; };
6206
6207         $oldtimeout = alarm($timeout);
6208
6209         my $parser = sub {
6210             my $line = shift;
6211
6212             print "$line\n";
6213
6214             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6215                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6216                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6217             } elsif ($line =~ m/^CTIME: /) {
6218                 # we correctly received the vma config, so we can disable
6219                 # the timeout now for disk allocation (set to 10 minutes, so
6220                 # that we always timeout if something goes wrong)
6221                 alarm(600);
6222                 &$print_devmap();
6223                 print $fifofh "done\n";
6224                 my $tmp = $oldtimeout || 0;
6225                 $oldtimeout = undef;
6226                 alarm($tmp);
6227                 close($fifofh);
6228             }
6229         };
6230
6231         print "restore vma archive: $dbg_cmdstring\n";
6232         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6233     };
6234     my $err = $@;
6235
6236     alarm($oldtimeout) if $oldtimeout;
6237
6238     my $vollist = [];
6239     foreach my $devname (keys %$devinfo) {
6240         my $volid = $devinfo->{$devname}->{volid};
6241         push @$vollist, $volid if $volid;
6242     }
6243
6244     PVE::Storage::deactivate_volumes($cfg, $vollist);
6245
6246     unlink $mapfifo;
6247
6248     if ($err) {
6249         rmtree $tmpdir;
6250         unlink $tmpfn;
6251
6252         foreach my $devname (keys %$devinfo) {
6253             my $volid = $devinfo->{$devname}->{volid};
6254             next if !$volid;
6255             eval {
6256                 if ($volid =~ m|^/|) {
6257                     unlink $volid || die 'unlink failed\n';
6258                 } else {
6259                     PVE::Storage::vdisk_free($cfg, $volid);
6260                 }
6261                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6262             };
6263             print STDERR "unable to cleanup '$volid' - $@" if $@;
6264         }
6265         die $err;
6266     }
6267
6268     rmtree $tmpdir;
6269
6270     rename($tmpfn, $conffile) ||
6271         die "unable to commit configuration file '$conffile'\n";
6272
6273     PVE::Cluster::cfs_update(); # make sure we read new file
6274
6275     eval { rescan($vmid, 1); };
6276     warn $@ if $@;
6277 }
6278
6279 sub restore_tar_archive {
6280     my ($archive, $vmid, $user, $opts) = @_;
6281
6282     if ($archive ne '-') {
6283         my $firstfile = tar_archive_read_firstfile($archive);
6284         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6285             if $firstfile ne 'qemu-server.conf';
6286     }
6287
6288     my $storecfg = PVE::Storage::config();
6289
6290     # destroy existing data - keep empty config
6291     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6292     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6293
6294     my $tocmd = "/usr/lib/qemu-server/qmextract";
6295
6296     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6297     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6298     $tocmd .= ' --prealloc' if $opts->{prealloc};
6299     $tocmd .= ' --info' if $opts->{info};
6300
6301     # tar option "xf" does not autodetect compression when read from STDIN,
6302     # so we pipe to zcat
6303     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6304         PVE::Tools::shellquote("--to-command=$tocmd");
6305
6306     my $tmpdir = "/var/tmp/vzdumptmp$$";
6307     mkpath $tmpdir;
6308
6309     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6310     local $ENV{VZDUMP_VMID} = $vmid;
6311     local $ENV{VZDUMP_USER} = $user;
6312
6313     my $conffile = PVE::QemuConfig->config_file($vmid);
6314     my $tmpfn = "$conffile.$$.tmp";
6315
6316     # disable interrupts (always do cleanups)
6317     local $SIG{INT} =
6318         local $SIG{TERM} =
6319         local $SIG{QUIT} =
6320         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6321
6322     eval {
6323         # enable interrupts
6324         local $SIG{INT} =
6325             local $SIG{TERM} =
6326             local $SIG{QUIT} =
6327             local $SIG{HUP} =
6328             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6329
6330         if ($archive eq '-') {
6331             print "extracting archive from STDIN\n";
6332             run_command($cmd, input => "<&STDIN");
6333         } else {
6334             print "extracting archive '$archive'\n";
6335             run_command($cmd);
6336         }
6337
6338         return if $opts->{info};
6339
6340         # read new mapping
6341         my $map = {};
6342         my $statfile = "$tmpdir/qmrestore.stat";
6343         if (my $fd = IO::File->new($statfile, "r")) {
6344             while (defined (my $line = <$fd>)) {
6345                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6346                     $map->{$1} = $2 if $1;
6347                 } else {
6348                     print STDERR "unable to parse line in statfile - $line\n";
6349                 }
6350             }
6351             $fd->close();
6352         }
6353
6354         my $confsrc = "$tmpdir/qemu-server.conf";
6355
6356         my $srcfd = new IO::File($confsrc, "r") ||
6357             die "unable to open file '$confsrc'\n";
6358
6359         my $outfd = new IO::File ($tmpfn, "w") ||
6360             die "unable to write config for VM $vmid\n";
6361
6362         my $cookie = { netcount => 0 };
6363         while (defined (my $line = <$srcfd>)) {
6364             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6365         }
6366
6367         $srcfd->close();
6368         $outfd->close();
6369     };
6370     my $err = $@;
6371
6372     if ($err) {
6373
6374         unlink $tmpfn;
6375
6376         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6377
6378         die $err;
6379     }
6380
6381     rmtree $tmpdir;
6382
6383     rename $tmpfn, $conffile ||
6384         die "unable to commit configuration file '$conffile'\n";
6385
6386     PVE::Cluster::cfs_update(); # make sure we read new file
6387
6388     eval { rescan($vmid, 1); };
6389     warn $@ if $@;
6390 };
6391
6392 sub foreach_storage_used_by_vm {
6393     my ($conf, $func) = @_;
6394
6395     my $sidhash = {};
6396
6397     foreach_drive($conf, sub {
6398         my ($ds, $drive) = @_;
6399         return if drive_is_cdrom($drive);
6400
6401         my $volid = $drive->{file};
6402
6403         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6404         $sidhash->{$sid} = $sid if $sid;
6405     });
6406
6407     foreach my $sid (sort keys %$sidhash) {
6408         &$func($sid);
6409     }
6410 }
6411
6412 sub do_snapshots_with_qemu {
6413     my ($storecfg, $volid) = @_;
6414
6415     my $storage_name = PVE::Storage::parse_volume_id($volid);
6416
6417     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6418         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6419         return 1;
6420     }
6421
6422     if ($volid =~ m/\.(qcow2|qed)$/){
6423         return 1;
6424     }
6425
6426     return undef;
6427 }
6428
6429 sub qga_check_running {
6430     my ($vmid, $nowarn) = @_;
6431
6432     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6433     if ($@) {
6434         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6435         return 0;
6436     }
6437     return 1;
6438 }
6439
6440 sub template_create {
6441     my ($vmid, $conf, $disk) = @_;
6442
6443     my $storecfg = PVE::Storage::config();
6444
6445     foreach_drive($conf, sub {
6446         my ($ds, $drive) = @_;
6447
6448         return if drive_is_cdrom($drive);
6449         return if $disk && $ds ne $disk;
6450
6451         my $volid = $drive->{file};
6452         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6453
6454         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6455         $drive->{file} = $voliddst;
6456         $conf->{$ds} = print_drive($vmid, $drive);
6457         PVE::QemuConfig->write_config($vmid, $conf);
6458     });
6459 }
6460
6461 sub qemu_img_convert {
6462     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6463
6464     my $storecfg = PVE::Storage::config();
6465     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6466     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6467
6468     if ($src_storeid && $dst_storeid) {
6469
6470         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6471
6472         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6473         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6474
6475         my $src_format = qemu_img_format($src_scfg, $src_volname);
6476         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6477
6478         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6479         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6480
6481         my $cmd = [];
6482         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6483         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6484         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6485         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6486         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6487         if ($is_zero_initialized) {
6488             push @$cmd, "zeroinit:$dst_path";
6489         } else {
6490             push @$cmd, $dst_path;
6491         }
6492
6493         my $parser = sub {
6494             my $line = shift;
6495             if($line =~ m/\((\S+)\/100\%\)/){
6496                 my $percent = $1;
6497                 my $transferred = int($size * $percent / 100);
6498                 my $remaining = $size - $transferred;
6499
6500                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6501             }
6502
6503         };
6504
6505         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6506         my $err = $@;
6507         die "copy failed: $err" if $err;
6508     }
6509 }
6510
6511 sub qemu_img_format {
6512     my ($scfg, $volname) = @_;
6513
6514     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6515         return $1;
6516     } else {
6517         return "raw";
6518     }
6519 }
6520
6521 sub qemu_drive_mirror {
6522     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6523
6524     $jobs = {} if !$jobs;
6525
6526     my $qemu_target;
6527     my $format;
6528     $jobs->{"drive-$drive"} = {};
6529
6530     if ($dst_volid =~ /^nbd:/) {
6531         $qemu_target = $dst_volid;
6532         $format = "nbd";
6533     } else {
6534         my $storecfg = PVE::Storage::config();
6535         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6536
6537         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6538
6539         $format = qemu_img_format($dst_scfg, $dst_volname);
6540
6541         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6542
6543         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6544     }
6545
6546     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6547     $opts->{format} = $format if $format;
6548
6549     print "drive mirror is starting for drive-$drive\n";
6550
6551     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6552
6553     if (my $err = $@) {
6554         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6555         die "mirroring error: $err";
6556     }
6557
6558     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6559 }
6560
6561 sub qemu_drive_mirror_monitor {
6562     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6563
6564     eval {
6565         my $err_complete = 0;
6566
6567         while (1) {
6568             die "storage migration timed out\n" if $err_complete > 300;
6569
6570             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6571
6572             my $running_mirror_jobs = {};
6573             foreach my $stat (@$stats) {
6574                 next if $stat->{type} ne 'mirror';
6575                 $running_mirror_jobs->{$stat->{device}} = $stat;
6576             }
6577
6578             my $readycounter = 0;
6579
6580             foreach my $job (keys %$jobs) {
6581
6582                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6583                     print "$job : finished\n";
6584                     delete $jobs->{$job};
6585                     next;
6586                 }
6587
6588                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6589
6590                 my $busy = $running_mirror_jobs->{$job}->{busy};
6591                 my $ready = $running_mirror_jobs->{$job}->{ready};
6592                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6593                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6594                     my $remaining = $total - $transferred;
6595                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6596
6597                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6598                 }
6599
6600                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6601             }
6602
6603             last if scalar(keys %$jobs) == 0;
6604
6605             if ($readycounter == scalar(keys %$jobs)) {
6606                 print "all mirroring jobs are ready \n";
6607                 last if $skipcomplete; #do the complete later
6608
6609                 if ($vmiddst && $vmiddst != $vmid) {
6610                     my $agent_running = $qga && qga_check_running($vmid);
6611                     if ($agent_running) {
6612                         print "freeze filesystem\n";
6613                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6614                     } else {
6615                         print "suspend vm\n";
6616                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6617                     }
6618
6619                     # if we clone a disk for a new target vm, we don't switch the disk
6620                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6621
6622                     if ($agent_running) {
6623                         print "unfreeze filesystem\n";
6624                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6625                     } else {
6626                         print "resume vm\n";
6627                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6628                     }
6629
6630                     last;
6631                 } else {
6632
6633                     foreach my $job (keys %$jobs) {
6634                         # try to switch the disk if source and destination are on the same guest
6635                         print "$job: Completing block job...\n";
6636
6637                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6638                         if ($@ =~ m/cannot be completed/) {
6639                             print "$job: Block job cannot be completed, try again.\n";
6640                             $err_complete++;
6641                         }else {
6642                             print "$job: Completed successfully.\n";
6643                             $jobs->{$job}->{complete} = 1;
6644                         }
6645                     }
6646                 }
6647             }
6648             sleep 1;
6649         }
6650     };
6651     my $err = $@;
6652
6653     if ($err) {
6654         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6655         die "mirroring error: $err";
6656     }
6657
6658 }
6659
6660 sub qemu_blockjobs_cancel {
6661     my ($vmid, $jobs) = @_;
6662
6663     foreach my $job (keys %$jobs) {
6664         print "$job: Cancelling block job\n";
6665         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6666         $jobs->{$job}->{cancel} = 1;
6667     }
6668
6669     while (1) {
6670         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6671
6672         my $running_jobs = {};
6673         foreach my $stat (@$stats) {
6674             $running_jobs->{$stat->{device}} = $stat;
6675         }
6676
6677         foreach my $job (keys %$jobs) {
6678
6679             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6680                 print "$job: Done.\n";
6681                 delete $jobs->{$job};
6682             }
6683         }
6684
6685         last if scalar(keys %$jobs) == 0;
6686
6687         sleep 1;
6688     }
6689 }
6690
6691 sub clone_disk {
6692     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6693         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6694
6695     my $newvolid;
6696
6697     if (!$full) {
6698         print "create linked clone of drive $drivename ($drive->{file})\n";
6699         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6700         push @$newvollist, $newvolid;
6701     } else {
6702
6703         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6704         $storeid = $storage if $storage;
6705
6706         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6707         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6708
6709         print "create full clone of drive $drivename ($drive->{file})\n";
6710         my $name = undef;
6711         if (drive_is_cloudinit($drive)) {
6712             $name = "vm-$newvmid-cloudinit";
6713             # cloudinit only supports raw and qcow2 atm:
6714             if ($dst_format eq 'qcow2') {
6715                 $name .= '.qcow2';
6716             } elsif ($dst_format ne 'raw') {
6717                 die "clone: unhandled format for cloudinit image\n";
6718             }
6719         }
6720         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6721         push @$newvollist, $newvolid;
6722
6723         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6724
6725         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6726         if (!$running || $snapname) {
6727             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6728         } else {
6729
6730             my $kvmver = get_running_qemu_version ($vmid);
6731             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6732                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6733                     if $drive->{iothread};
6734             }
6735
6736             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6737         }
6738     }
6739
6740     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6741
6742     my $disk = $drive;
6743     $disk->{format} = undef;
6744     $disk->{file} = $newvolid;
6745     $disk->{size} = $size;
6746
6747     return $disk;
6748 }
6749
6750 # this only works if VM is running
6751 sub get_current_qemu_machine {
6752     my ($vmid) = @_;
6753
6754     my $cmd = { execute => 'query-machines', arguments => {} };
6755     my $res = vm_qmp_command($vmid, $cmd);
6756
6757     my ($current, $default);
6758     foreach my $e (@$res) {
6759         $default = $e->{name} if $e->{'is-default'};
6760         $current = $e->{name} if $e->{'is-current'};
6761     }
6762
6763     # fallback to the default machine if current is not supported by qemu
6764     return $current || $default || 'pc';
6765 }
6766
6767 sub get_running_qemu_version {
6768     my ($vmid) = @_;
6769     my $cmd = { execute => 'query-version', arguments => {} };
6770     my $res = vm_qmp_command($vmid, $cmd);
6771     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6772 }
6773
6774 sub qemu_machine_feature_enabled {
6775     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6776
6777     my $current_major;
6778     my $current_minor;
6779
6780     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
6781
6782         $current_major = $3;
6783         $current_minor = $4;
6784
6785     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6786
6787         $current_major = $1;
6788         $current_minor = $2;
6789     }
6790
6791     return 1 if $current_major > $version_major ||
6792                 ($current_major == $version_major &&
6793                  $current_minor >= $version_minor);
6794 }
6795
6796 sub qemu_machine_pxe {
6797     my ($vmid, $conf, $machine) = @_;
6798
6799     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6800
6801     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6802         $machine .= '.pxe';
6803     }
6804
6805     return $machine;
6806 }
6807
6808 sub qemu_use_old_bios_files {
6809     my ($machine_type) = @_;
6810
6811     return if !$machine_type;
6812
6813     my $use_old_bios_files = undef;
6814
6815     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6816         $machine_type = $1;
6817         $use_old_bios_files = 1;
6818     } else {
6819         my $kvmver = kvm_user_version();
6820         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6821         # load new efi bios files on migration. So this hack is required to allow
6822         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6823         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6824         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6825     }
6826
6827     return ($use_old_bios_files, $machine_type);
6828 }
6829
6830 sub create_efidisk($$$$$) {
6831     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6832
6833     my (undef, $ovmf_vars) = get_ovmf_files($arch);
6834     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6835
6836     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
6837     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6838     PVE::Storage::activate_volumes($storecfg, [$volid]);
6839
6840     my $path = PVE::Storage::path($storecfg, $volid);
6841     eval {
6842         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
6843     };
6844     die "Copying EFI vars image failed: $@" if $@;
6845
6846     return ($volid, $vars_size);
6847 }
6848
6849 sub lspci {
6850
6851     my $devices = {};
6852
6853     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6854             my (undef, $id, $function) = @_;
6855             my $res = { id => $id, function => $function};
6856             push @{$devices->{$id}}, $res;
6857     });
6858
6859     # Entries should be sorted by functions.
6860     foreach my $id (keys %$devices) {
6861         my $dev = $devices->{$id};
6862         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6863     }
6864
6865     return $devices;
6866 }
6867
6868 sub vm_iothreads_list {
6869     my ($vmid) = @_;
6870
6871     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6872
6873     my $iothreads = {};
6874     foreach my $iothread (@$res) {
6875         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6876     }
6877
6878     return $iothreads;
6879 }
6880
6881 sub scsihw_infos {
6882     my ($conf, $drive) = @_;
6883
6884     my $maxdev = 0;
6885
6886     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6887         $maxdev = 7;
6888     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6889         $maxdev = 1;
6890     } else {
6891         $maxdev = 256;
6892     }
6893
6894     my $controller = int($drive->{index} / $maxdev);
6895     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6896
6897     return ($maxdev, $controller, $controller_prefix);
6898 }
6899
6900 sub add_hyperv_enlightenments {
6901     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6902
6903     return if $winversion < 6;
6904     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6905
6906     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6907
6908     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6909         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6910         push @$cpuFlags , 'hv_vapic';
6911         push @$cpuFlags , 'hv_time';
6912     } else {
6913         push @$cpuFlags , 'hv_spinlocks=0xffff';
6914     }
6915
6916     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6917         push @$cpuFlags , 'hv_reset';
6918         push @$cpuFlags , 'hv_vpindex';
6919         push @$cpuFlags , 'hv_runtime';
6920     }
6921
6922     if ($winversion >= 7) {
6923         push @$cpuFlags , 'hv_relaxed';
6924
6925         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6926             push @$cpuFlags , 'hv_synic';
6927             push @$cpuFlags , 'hv_stimer';
6928         }
6929     }
6930 }
6931
6932 sub windows_version {
6933     my ($ostype) = @_;
6934
6935     return 0 if !$ostype;
6936
6937     my $winversion = 0;
6938
6939     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6940         $winversion = 5;
6941     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6942         $winversion = 6;
6943     } elsif ($ostype =~ m/^win(\d+)$/) {
6944         $winversion = $1;
6945     }
6946
6947     return $winversion;
6948 }
6949
6950 sub resolve_dst_disk_format {
6951         my ($storecfg, $storeid, $src_volname, $format) = @_;
6952         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6953
6954         if (!$format) {
6955             # if no target format is specified, use the source disk format as hint
6956             if ($src_volname) {
6957                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6958                 $format = qemu_img_format($scfg, $src_volname);
6959             } else {
6960                 return $defFormat;
6961             }
6962         }
6963
6964         # test if requested format is supported - else use default
6965         my $supported = grep { $_ eq $format } @$validFormats;
6966         $format = $defFormat if !$supported;
6967         return $format;
6968 }
6969
6970 sub resolve_first_disk {
6971     my $conf = shift;
6972     my @disks = PVE::QemuServer::valid_drive_names();
6973     my $firstdisk;
6974     foreach my $ds (reverse @disks) {
6975         next if !$conf->{$ds};
6976         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6977         next if PVE::QemuServer::drive_is_cdrom($disk);
6978         $firstdisk = $ds;
6979     }
6980     return $firstdisk;
6981 }
6982
6983 sub generate_uuid {
6984     my ($uuid, $uuid_str);
6985     UUID::generate($uuid);
6986     UUID::unparse($uuid, $uuid_str);
6987     return $uuid_str;
6988 }
6989
6990 sub generate_smbios1_uuid {
6991     return "uuid=".generate_uuid();
6992 }
6993
6994 sub nbd_stop {
6995     my ($vmid) = @_;
6996
6997     vm_mon_cmd($vmid, 'nbd-server-stop');
6998 }
6999
7000 # bash completion helper
7001
7002 sub complete_backup_archives {
7003     my ($cmdname, $pname, $cvalue) = @_;
7004
7005     my $cfg = PVE::Storage::config();
7006
7007     my $storeid;
7008
7009     if ($cvalue =~ m/^([^:]+):/) {
7010         $storeid = $1;
7011     }
7012
7013     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7014
7015     my $res = [];
7016     foreach my $id (keys %$data) {
7017         foreach my $item (@{$data->{$id}}) {
7018             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7019             push @$res, $item->{volid} if defined($item->{volid});
7020         }
7021     }
7022
7023     return $res;
7024 }
7025
7026 my $complete_vmid_full = sub {
7027     my ($running) = @_;
7028
7029     my $idlist = vmstatus();
7030
7031     my $res = [];
7032
7033     foreach my $id (keys %$idlist) {
7034         my $d = $idlist->{$id};
7035         if (defined($running)) {
7036             next if $d->{template};
7037             next if $running && $d->{status} ne 'running';
7038             next if !$running && $d->{status} eq 'running';
7039         }
7040         push @$res, $id;
7041
7042     }
7043     return $res;
7044 };
7045
7046 sub complete_vmid {
7047     return &$complete_vmid_full();
7048 }
7049
7050 sub complete_vmid_stopped {
7051     return &$complete_vmid_full(0);
7052 }
7053
7054 sub complete_vmid_running {
7055     return &$complete_vmid_full(1);
7056 }
7057
7058 sub complete_storage {
7059
7060     my $cfg = PVE::Storage::config();
7061     my $ids = $cfg->{ids};
7062
7063     my $res = [];
7064     foreach my $sid (keys %$ids) {
7065         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7066         next if !$ids->{$sid}->{content}->{images};
7067         push @$res, $sid;
7068     }
7069
7070     return $res;
7071 }
7072
7073 1;