PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $confdesc = {
 227     onboot => {
 228         optional => 1,
 229         type => 'boolean',
 230         description => "Specifies whether a VM will be started during system bootup.",
 231         default => 0,
 232     },
 233     autostart => {
 234         optional => 1,
 235         type => 'boolean',
 236         description => "Automatic restart after crash (currently ignored).",
 237         default => 0,
 238     },
 239     hotplug => {
 240         optional => 1,
 241         type => 'string', format => 'pve-hotplug-features',
 242         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 243         default => 'network,disk,usb',
 244     },
 245     reboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 249         default => 1,
 250     },
 251     lock => {
 252         optional => 1,
 253         type => 'string',
 254         description => "Lock/unlock the VM.",
 255         enum => [qw(migrate backup snapshot rollback)],
 256     },
 257     cpulimit => {
 258         optional => 1,
 259         type => 'number',
 260         description => "Limit of CPU usage.",
 261         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 262         minimum => 0,
 263         maximum => 128,
 264         default => 0,
 265     },
 266     cpuunits => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "CPU weight for a VM.",
 270         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 271         minimum => 2,
 272         maximum => 262144,
 273         default => 1024,
 274     },
 275     memory => {
 276         optional => 1,
 277         type => 'integer',
 278         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 279         minimum => 16,
 280         default => 512,
 281     },
 282     balloon => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 286         minimum => 0,
 287     },
 288     shares => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 292         minimum => 0,
 293         maximum => 50000,
 294         default => 1000,
 295     },
 296     keyboard => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 300                        "It should not be necessary to set it.",
 301         enum => PVE::Tools::kvmkeymaplist(),
 302         default => undef,
 303     },
 304     name => {
 305         optional => 1,
 306         type => 'string', format => 'dns-name',
 307         description => "Set a name for the VM. Only used on the configuration web interface.",
 308     },
 309     scsihw => {
 310         optional => 1,
 311         type => 'string',
 312         description => "SCSI controller model",
 313         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 314         default => 'lsi',
 315     },
 316     description => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 320     },
 321     ostype => {
 322         optional => 1,
 323         type => 'string',
 324         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 325         description => "Specify guest operating system.",
 326         verbose_description => <<EODESC,
 327 Specify guest operating system. This is used to enable special
 328 optimization/features for specific operating systems:
 329
 330 [horizontal]
 331 other;; unspecified OS
 332 wxp;; Microsoft Windows XP
 333 w2k;; Microsoft Windows 2000
 334 w2k3;; Microsoft Windows 2003
 335 w2k8;; Microsoft Windows 2008
 336 wvista;; Microsoft Windows Vista
 337 win7;; Microsoft Windows 7
 338 win8;; Microsoft Windows 8/2012/2012r2
 339 win10;; Microsoft Windows 10/2016
 340 l24;; Linux 2.4 Kernel
 341 l26;; Linux 2.6/3.X Kernel
 342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 343 EODESC
 344     },
 345     boot => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 349         pattern => '[acdn]{1,4}',
 350         default => 'cdn',
 351     },
 352     bootdisk => {
 353         optional => 1,
 354         type => 'string', format => 'pve-qm-bootdisk',
 355         description => "Enable booting from specified disk.",
 356         pattern => '(ide|sata|scsi|virtio)\d+',
 357     },
 358     smp => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "The number of CPUs. Please use option -sockets instead.",
 362         minimum => 1,
 363         default => 1,
 364     },
 365     sockets => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "The number of CPU sockets.",
 369         minimum => 1,
 370         default => 1,
 371     },
 372     cores => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "The number of cores per socket.",
 376         minimum => 1,
 377         default => 1,
 378     },
 379     numa => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable NUMA.",
 383         default => 0,
 384     },
 385     hugepages => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Enable/disable hugepages memory.",
 389         enum => [qw(any 2 1024)],
 390     },
 391     vcpus => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Number of hotplugged vcpus.",
 395         minimum => 1,
 396         default => 0,
 397     },
 398     acpi => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Enable/disable ACPI.",
 402         default => 1,
 403     },
 404     agent => {
 405         optional => 1,
 406         description => "Enable/disable Qemu GuestAgent and its properties.",
 407         type => 'string',
 408         format => $agent_fmt,
 409     },
 410     kvm => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable KVM hardware virtualization.",
 414         default => 1,
 415     },
 416     tdf => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable time drift fix.",
 420         default => 0,
 421     },
 422     localtime => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 426     },
 427     freeze => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 431     },
 432     vga => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Select the VGA type.",
 436         verbose_description => "Select the VGA type. If you want to use high resolution" .
 437             " modes (>= 1280x1024x16) then you should use the options " .
 438             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 439             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 440             "display sever. For win* OS you can select how many independent " .
 441             "displays you want, Linux guests can add displays them self. " .
 442             "You can also run without any graphic card, using a serial device" .
 443             " as terminal.",
 444         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 445     },
 446     watchdog => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-watchdog',
 449         description => "Create a virtual hardware watchdog device.",
 450         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 451             " (by a guest action), the watchdog must be periodically polled " .
 452             "by an agent inside the guest or else the watchdog will reset " .
 453             "the guest (or execute the respective action specified)",
 454     },
 455     startdate => {
 456         optional => 1,
 457         type => 'string',
 458         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 459         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 460         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 461         default => 'now',
 462     },
 463     startup =>  get_standard_option('pve-startup-order'),
 464     template => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable Template.",
 468         default => 0,
 469     },
 470     args => {
 471         optional => 1,
 472         type => 'string',
 473         description => "Arbitrary arguments passed to kvm.",
 474         verbose_description => <<EODESCR,
 475 Arbitrary arguments passed to kvm, for example:
 476
 477 args: -no-reboot -no-hpet
 478
 479 NOTE: this option is for experts only.
 480 EODESCR
 481     },
 482     tablet => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 1,
 486         description => "Enable/disable the USB tablet device.",
 487         verbose_description => "Enable/disable the USB tablet device. This device is " .
 488             "usually needed to allow absolute mouse positioning with VNC. " .
 489             "Else the mouse runs out of sync with normal VNC clients. " .
 490             "If you're running lots of console-only guests on one host, " .
 491             "you may consider disabling this to save some context switches. " .
 492             "This is turned off by default if you use spice (-vga=qxl).",
 493     },
 494     migrate_speed => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 498         minimum => 0,
 499         default => 0,
 500     },
 501     migrate_downtime => {
 502         optional => 1,
 503         type => 'number',
 504         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 505         minimum => 0,
 506         default => 0.1,
 507     },
 508     cdrom => {
 509         optional => 1,
 510         type => 'string', format => 'pve-qm-ide',
 511         typetext => '<volume>',
 512         description => "This is an alias for option -ide2",
 513     },
 514     cpu => {
 515         optional => 1,
 516         description => "Emulated CPU type.",
 517         type => 'string',
 518         format => $cpu_fmt,
 519     },
 520     parent => get_standard_option('pve-snapshot-name', {
 521         optional => 1,
 522         description => "Parent snapshot name. This is used internally, and should not be modified.",
 523     }),
 524     snaptime => {
 525         optional => 1,
 526         description => "Timestamp for snapshots.",
 527         type => 'integer',
 528         minimum => 0,
 529     },
 530     vmstate => {
 531         optional => 1,
 532         type => 'string', format => 'pve-volume-id',
 533         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 534     },
 535     vmstatestorage => get_standard_option('pve-storage-id', {
 536         description => "Default storage for VM state volumes/files.",
 537         optional => 1,
 538     }),
 539     runningmachine => get_standard_option('pve-qemu-machine', {
 540         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 541     }),
 542     machine => get_standard_option('pve-qemu-machine'),
 543     smbios1 => {
 544         description => "Specify SMBIOS type 1 fields.",
 545         type => 'string', format => 'pve-qm-smbios1',
 546         maxLength => 256,
 547         optional => 1,
 548     },
 549     protection => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 553         default => 0,
 554     },
 555     bios => {
 556         optional => 1,
 557         type => 'string',
 558         enum => [ qw(seabios ovmf) ],
 559         description => "Select BIOS implementation.",
 560         default => 'seabios',
 561     },
 562     vmgenid => {
 563         type => 'string',
 564         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 565         format_description => 'UUID',
 566         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 567         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 568             " 128-bit integer value identifier to the guest OS. This allows to".
 569             " notify the guest operating system when the virtual machine is".
 570             " executed with a different configuration (e.g. snapshot execution".
 571             " or creation from a template). The guest operating system notices".
 572             " the change, and is then able to react as appropriate by marking".
 573             " its copies of distributed databases as dirty, re-initializing its".
 574             " random number generator, etc.\n".
 575             "Note that auto-creation only works when done throug API/CLI create".
 576             " or update methods, but not when manually editing the config file.",
 577         default => "1 (autogenerated)",
 578         optional => 1,
 579     },
 580 };
 581
 582 my $confdesc_cloudinit = {
 583     citype => {
 584         optional => 1,
 585         type => 'string',
 586         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 587         enum => ['configdrive2', 'nocloud'],
 588     },
 589     ciuser => {
 590         optional => 1,
 591         type => 'string',
 592         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 593     },
 594     cipassword => {
 595         optional => 1,
 596         type => 'string',
 597         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 598     },
 599     searchdomain => {
 600         optional => 1,
 601         type => 'string',
 602         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 603     },
 604     nameserver => {
 605         optional => 1,
 606         type => 'string', format => 'address-list',
 607         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 608     },
 609     sshkeys => {
 610         optional => 1,
 611         type => 'string',
 612         format => 'urlencoded',
 613         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 614     },
 615 };
 616
 617 # what about other qemu settings ?
 618 #cpu => 'string',
 619 #machine => 'string',
 620 #fda => 'file',
 621 #fdb => 'file',
 622 #mtdblock => 'file',
 623 #sd => 'file',
 624 #pflash => 'file',
 625 #snapshot => 'bool',
 626 #bootp => 'file',
 627 ##tftp => 'dir',
 628 ##smb => 'dir',
 629 #kernel => 'file',
 630 #append => 'string',
 631 #initrd => 'file',
 632 ##soundhw => 'string',
 633
 634 while (my ($k, $v) = each %$confdesc) {
 635     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 636 }
 637
 638 my $MAX_IDE_DISKS = 4;
 639 my $MAX_SCSI_DISKS = 14;
 640 my $MAX_VIRTIO_DISKS = 16;
 641 my $MAX_SATA_DISKS = 6;
 642 my $MAX_USB_DEVICES = 5;
 643 my $MAX_NETS = 32;
 644 my $MAX_UNUSED_DISKS = 8;
 645 my $MAX_HOSTPCI_DEVICES = 4;
 646 my $MAX_SERIAL_PORTS = 4;
 647 my $MAX_PARALLEL_PORTS = 3;
 648 my $MAX_NUMA = 8;
 649
 650 my $numa_fmt = {
 651     cpus => {
 652         type => "string",
 653         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 654         description => "CPUs accessing this NUMA node.",
 655         format_description => "id[-id];...",
 656     },
 657     memory => {
 658         type => "number",
 659         description => "Amount of memory this NUMA node provides.",
 660         optional => 1,
 661     },
 662     hostnodes => {
 663         type => "string",
 664         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 665         description => "Host NUMA nodes to use.",
 666         format_description => "id[-id];...",
 667         optional => 1,
 668     },
 669     policy => {
 670         type => 'string',
 671         enum => [qw(preferred bind interleave)],
 672         description => "NUMA allocation policy.",
 673         optional => 1,
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 677 my $numadesc = {
 678     optional => 1,
 679     type => 'string', format => $numa_fmt,
 680     description => "NUMA topology.",
 681 };
 682 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 683
 684 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 685     $confdesc->{"numa$i"} = $numadesc;
 686 }
 687
 688 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 689                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 690                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 691 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 692
 693 my $net_fmt_bridge_descr = <<__EOD__;
 694 Bridge to attach the network device to. The Proxmox VE standard bridge
 695 is called 'vmbr0'.
 696
 697 If you do not specify a bridge, we create a kvm user (NATed) network
 698 device, which provides DHCP and DNS services. The following addresses
 699 are used:
 700
 701  10.0.2.2   Gateway
 702  10.0.2.3   DNS Server
 703  10.0.2.4   SMB Server
 704
 705 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 706 __EOD__
 707
 708 my $net_fmt = {
 709     macaddr => {
 710         type => 'string',
 711         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 712         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 713         format_description => "XX:XX:XX:XX:XX:XX",
 714         optional => 1,
 715     },
 716     model => {
 717         type => 'string',
 718         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 719         enum => $nic_model_list,
 720         default_key => 1,
 721     },
 722     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 723     bridge => {
 724         type => 'string',
 725         description => $net_fmt_bridge_descr,
 726         format_description => 'bridge',
 727         optional => 1,
 728     },
 729     queues => {
 730         type => 'integer',
 731         minimum => 0, maximum => 16,
 732         description => 'Number of packet queues to be used on the device.',
 733         optional => 1,
 734     },
 735     rate => {
 736         type => 'number',
 737         minimum => 0,
 738         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 739         optional => 1,
 740     },
 741     tag => {
 742         type => 'integer',
 743         minimum => 1, maximum => 4094,
 744         description => 'VLAN tag to apply to packets on this interface.',
 745         optional => 1,
 746     },
 747     trunks => {
 748         type => 'string',
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => 'VLAN trunks to pass through this interface.',
 751         format_description => 'vlanid[;vlanid...]',
 752         optional => 1,
 753     },
 754     firewall => {
 755         type => 'boolean',
 756         description => 'Whether this interface should be protected by the firewall.',
 757         optional => 1,
 758     },
 759     link_down => {
 760         type => 'boolean',
 761         description => 'Whether this interface should be disconnected (like pulling the plug).',
 762         optional => 1,
 763     },
 764 };
 765
 766 my $netdesc = {
 767     optional => 1,
 768     type => 'string', format => $net_fmt,
 769     description => "Specify network devices.",
 770 };
 771
 772 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 773
 774 my $ipconfig_fmt = {
 775     ip => {
 776         type => 'string',
 777         format => 'pve-ipv4-config',
 778         format_description => 'IPv4Format/CIDR',
 779         description => 'IPv4 address in CIDR format.',
 780         optional => 1,
 781         default => 'dhcp',
 782     },
 783     gw => {
 784         type => 'string',
 785         format => 'ipv4',
 786         format_description => 'GatewayIPv4',
 787         description => 'Default gateway for IPv4 traffic.',
 788         optional => 1,
 789         requires => 'ip',
 790     },
 791     ip6 => {
 792         type => 'string',
 793         format => 'pve-ipv6-config',
 794         format_description => 'IPv6Format/CIDR',
 795         description => 'IPv6 address in CIDR format.',
 796         optional => 1,
 797         default => 'dhcp',
 798     },
 799     gw6 => {
 800         type => 'string',
 801         format => 'ipv6',
 802         format_description => 'GatewayIPv6',
 803         description => 'Default gateway for IPv6 traffic.',
 804         optional => 1,
 805         requires => 'ip6',
 806     },
 807 };
 808 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 809 my $ipconfigdesc = {
 810     optional => 1,
 811     type => 'string', format => 'pve-qm-ipconfig',
 812     description => <<'EODESCR',
 813 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 814
 815 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 816
 817 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 818 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 819
 820 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 821 EODESCR
 822 };
 823 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 824
 825 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 826     $confdesc->{"net$i"} = $netdesc;
 827     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 828 }
 829
 830 foreach my $key (keys %$confdesc_cloudinit) {
 831     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 832 }
 833
 834 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 835 sub verify_volume_id_or_qm_path {
 836     my ($volid, $noerr) = @_;
 837
 838     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 839         return $volid;
 840     }
 841
 842     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 843     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 844     if ($@) {
 845         return undef if $noerr;
 846         die $@;
 847     }
 848     return $volid;
 849 }
 850
 851 my $drivename_hash;
 852
 853 my %drivedesc_base = (
 854     volume => { alias => 'file' },
 855     file => {
 856         type => 'string',
 857         format => 'pve-volume-id-or-qm-path',
 858         default_key => 1,
 859         format_description => 'volume',
 860         description => "The drive's backing volume.",
 861     },
 862     media => {
 863         type => 'string',
 864         enum => [qw(cdrom disk)],
 865         description => "The drive's media type.",
 866         default => 'disk',
 867         optional => 1
 868     },
 869     cyls => {
 870         type => 'integer',
 871         description => "Force the drive's physical geometry to have a specific cylinder count.",
 872         optional => 1
 873     },
 874     heads => {
 875         type => 'integer',
 876         description => "Force the drive's physical geometry to have a specific head count.",
 877         optional => 1
 878     },
 879     secs => {
 880         type => 'integer',
 881         description => "Force the drive's physical geometry to have a specific sector count.",
 882         optional => 1
 883     },
 884     trans => {
 885         type => 'string',
 886         enum => [qw(none lba auto)],
 887         description => "Force disk geometry bios translation mode.",
 888         optional => 1,
 889     },
 890     snapshot => {
 891         type => 'boolean',
 892         description => "Controls qemu's snapshot mode feature."
 893             . " If activated, changes made to the disk are temporary and will"
 894             . " be discarded when the VM is shutdown.",
 895         optional => 1,
 896     },
 897     cache => {
 898         type => 'string',
 899         enum => [qw(none writethrough writeback unsafe directsync)],
 900         description => "The drive's cache mode",
 901         optional => 1,
 902     },
 903     format => get_standard_option('pve-qm-image-format'),
 904     size => {
 905         type => 'string',
 906         format => 'disk-size',
 907         format_description => 'DiskSize',
 908         description => "Disk size. This is purely informational and has no effect.",
 909         optional => 1,
 910     },
 911     backup => {
 912         type => 'boolean',
 913         description => "Whether the drive should be included when making backups.",
 914         optional => 1,
 915     },
 916     replicate => {
 917         type => 'boolean',
 918         description => 'Whether the drive should considered for replication jobs.',
 919         optional => 1,
 920         default => 1,
 921     },
 922     rerror => {
 923         type => 'string',
 924         enum => [qw(ignore report stop)],
 925         description => 'Read error action.',
 926         optional => 1,
 927     },
 928     werror => {
 929         type => 'string',
 930         enum => [qw(enospc ignore report stop)],
 931         description => 'Write error action.',
 932         optional => 1,
 933     },
 934     aio => {
 935         type => 'string',
 936         enum => [qw(native threads)],
 937         description => 'AIO type to use.',
 938         optional => 1,
 939     },
 940     discard => {
 941         type => 'string',
 942         enum => [qw(ignore on)],
 943         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 944         optional => 1,
 945     },
 946     detect_zeroes => {
 947         type => 'boolean',
 948         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 949         optional => 1,
 950     },
 951     serial => {
 952         type => 'string',
 953         format => 'urlencoded',
 954         format_description => 'serial',
 955         maxLength => 20*3, # *3 since it's %xx url enoded
 956         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 957         optional => 1,
 958     },
 959     shared => {
 960         type => 'boolean',
 961         description => 'Mark this locally-managed volume as available on all nodes',
 962         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 963         optional => 1,
 964         default => 0,
 965     }
 966 );
 967
 968 my %iothread_fmt = ( iothread => {
 969         type => 'boolean',
 970         description => "Whether to use iothreads for this drive",
 971         optional => 1,
 972 });
 973
 974 my %model_fmt = (
 975     model => {
 976         type => 'string',
 977         format => 'urlencoded',
 978         format_description => 'model',
 979         maxLength => 40*3, # *3 since it's %xx url enoded
 980         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 981         optional => 1,
 982     },
 983 );
 984
 985 my %queues_fmt = (
 986     queues => {
 987         type => 'integer',
 988         description => "Number of queues.",
 989         minimum => 2,
 990         optional => 1
 991     }
 992 );
 993
 994 my %scsiblock_fmt = (
 995     scsiblock => {
 996         type => 'boolean',
 997         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 998         optional => 1,
 999         default => 0,
1000     },
1001 );
1002
1003 my %ssd_fmt = (
1004     ssd => {
1005         type => 'boolean',
1006         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1007         optional => 1,
1008     },
1009 );
1010
1011 my $add_throttle_desc = sub {
1012     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1013     my $d = {
1014         type => $type,
1015         format_description => $unit,
1016         description => "Maximum $what in $longunit.",
1017         optional => 1,
1018     };
1019     $d->{minimum} = $minimum if defined($minimum);
1020     $drivedesc_base{$key} = $d;
1021 };
1022 # throughput: (leaky bucket)
1023 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1024 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1025 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1026 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1027 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1028 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1029 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1030 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1031 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1032
1033 # pools: (pool of IO before throttling starts taking effect)
1034 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1035 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1036 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1037 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1038 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1039 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1040
1041 # burst lengths
1042 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1043 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1044 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1045 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1046 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1047 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1048
1049 # legacy support
1050 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1051 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1052 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1053 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1054
1055 my $ide_fmt = {
1056     %drivedesc_base,
1057     %model_fmt,
1058     %ssd_fmt,
1059 };
1060 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1061
1062 my $idedesc = {
1063     optional => 1,
1064     type => 'string', format => $ide_fmt,
1065     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1066 };
1067 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1068
1069 my $scsi_fmt = {
1070     %drivedesc_base,
1071     %iothread_fmt,
1072     %queues_fmt,
1073     %scsiblock_fmt,
1074     %ssd_fmt,
1075 };
1076 my $scsidesc = {
1077     optional => 1,
1078     type => 'string', format => $scsi_fmt,
1079     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1080 };
1081 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1082
1083 my $sata_fmt = {
1084     %drivedesc_base,
1085     %ssd_fmt,
1086 };
1087 my $satadesc = {
1088     optional => 1,
1089     type => 'string', format => $sata_fmt,
1090     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1091 };
1092 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1093
1094 my $virtio_fmt = {
1095     %drivedesc_base,
1096     %iothread_fmt,
1097 };
1098 my $virtiodesc = {
1099     optional => 1,
1100     type => 'string', format => $virtio_fmt,
1101     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1102 };
1103 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1104
1105 my $alldrive_fmt = {
1106     %drivedesc_base,
1107     %iothread_fmt,
1108     %model_fmt,
1109     %queues_fmt,
1110     %scsiblock_fmt,
1111     %ssd_fmt,
1112 };
1113
1114 my $efidisk_fmt = {
1115     volume => { alias => 'file' },
1116     file => {
1117         type => 'string',
1118         format => 'pve-volume-id-or-qm-path',
1119         default_key => 1,
1120         format_description => 'volume',
1121         description => "The drive's backing volume.",
1122     },
1123     format => get_standard_option('pve-qm-image-format'),
1124     size => {
1125         type => 'string',
1126         format => 'disk-size',
1127         format_description => 'DiskSize',
1128         description => "Disk size. This is purely informational and has no effect.",
1129         optional => 1,
1130     },
1131 };
1132
1133 my $efidisk_desc = {
1134     optional => 1,
1135     type => 'string', format => $efidisk_fmt,
1136     description => "Configure a Disk for storing EFI vars",
1137 };
1138
1139 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1140
1141 my $usb_fmt = {
1142     host => {
1143         default_key => 1,
1144         type => 'string', format => 'pve-qm-usb-device',
1145         format_description => 'HOSTUSBDEVICE|spice',
1146         description => <<EODESCR,
1147 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1148
1149  'bus-port(.port)*' (decimal numbers) or
1150  'vendor_id:product_id' (hexadeciaml numbers) or
1151  'spice'
1152
1153 You can use the 'lsusb -t' command to list existing usb devices.
1154
1155 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1156
1157 The value 'spice' can be used to add a usb redirection devices for spice.
1158 EODESCR
1159     },
1160     usb3 => {
1161         optional => 1,
1162         type => 'boolean',
1163         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1164         default => 0,
1165     },
1166 };
1167
1168 my $usbdesc = {
1169     optional => 1,
1170     type => 'string', format => $usb_fmt,
1171     description => "Configure an USB device (n is 0 to 4).",
1172 };
1173 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1174
1175 # NOTE: the match-groups of this regex are used in parse_hostpci
1176 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1177 my $hostpci_fmt = {
1178     host => {
1179         default_key => 1,
1180         type => 'string',
1181         pattern => qr/$PCIRE(;$PCIRE)*/,
1182         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1183         description => <<EODESCR,
1184 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1185 of PCI virtual functions of the host. HOSTPCIID syntax is:
1186
1187 'bus:dev.func' (hexadecimal numbers)
1188
1189 You can us the 'lspci' command to list existing PCI devices.
1190 EODESCR
1191     },
1192     rombar => {
1193         type => 'boolean',
1194         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1195         optional => 1,
1196         default => 1,
1197     },
1198     romfile => {
1199         type => 'string',
1200         pattern => '[^,;]+',
1201         format_description => 'string',
1202         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1203         optional => 1,
1204     },
1205     pcie => {
1206         type => 'boolean',
1207         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1208         optional => 1,
1209         default => 0,
1210     },
1211     'x-vga' => {
1212         type => 'boolean',
1213         description =>  "Enable vfio-vga device support.",
1214         optional => 1,
1215         default => 0,
1216     },
1217 };
1218 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1219
1220 my $hostpcidesc = {
1221         optional => 1,
1222         type => 'string', format => 'pve-qm-hostpci',
1223         description => "Map host PCI devices into guest.",
1224         verbose_description =>  <<EODESCR,
1225 Map host PCI devices into guest.
1226
1227 NOTE: This option allows direct access to host hardware. So it is no longer
1228 possible to migrate such machines - use with special care.
1229
1230 CAUTION: Experimental! User reported problems with this option.
1231 EODESCR
1232 };
1233 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1234
1235 my $serialdesc = {
1236         optional => 1,
1237         type => 'string',
1238         pattern => '(/dev/.+|socket)',
1239         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1240         verbose_description =>  <<EODESCR,
1241 Create a serial device inside the VM (n is 0 to 3), and pass through a
1242 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1243 host side (use 'qm terminal' to open a terminal connection).
1244
1245 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1246
1247 CAUTION: Experimental! User reported problems with this option.
1248 EODESCR
1249 };
1250
1251 my $paralleldesc= {
1252         optional => 1,
1253         type => 'string',
1254         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1255         description =>  "Map host parallel devices (n is 0 to 2).",
1256         verbose_description =>  <<EODESCR,
1257 Map host parallel devices (n is 0 to 2).
1258
1259 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1260
1261 CAUTION: Experimental! User reported problems with this option.
1262 EODESCR
1263 };
1264
1265 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1266     $confdesc->{"parallel$i"} = $paralleldesc;
1267 }
1268
1269 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1270     $confdesc->{"serial$i"} = $serialdesc;
1271 }
1272
1273 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1274     $confdesc->{"hostpci$i"} = $hostpcidesc;
1275 }
1276
1277 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1278     $drivename_hash->{"ide$i"} = 1;
1279     $confdesc->{"ide$i"} = $idedesc;
1280 }
1281
1282 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1283     $drivename_hash->{"sata$i"} = 1;
1284     $confdesc->{"sata$i"} = $satadesc;
1285 }
1286
1287 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1288     $drivename_hash->{"scsi$i"} = 1;
1289     $confdesc->{"scsi$i"} = $scsidesc ;
1290 }
1291
1292 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1293     $drivename_hash->{"virtio$i"} = 1;
1294     $confdesc->{"virtio$i"} = $virtiodesc;
1295 }
1296
1297 $drivename_hash->{efidisk0} = 1;
1298 $confdesc->{efidisk0} = $efidisk_desc;
1299
1300 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1301     $confdesc->{"usb$i"} = $usbdesc;
1302 }
1303
1304 my $unuseddesc = {
1305     optional => 1,
1306     type => 'string', format => 'pve-volume-id',
1307     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1308 };
1309
1310 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1311     $confdesc->{"unused$i"} = $unuseddesc;
1312 }
1313
1314 my $kvm_api_version = 0;
1315
1316 sub kvm_version {
1317
1318     return $kvm_api_version if $kvm_api_version;
1319
1320     my $fh = IO::File->new("</dev/kvm") ||
1321         return 0;
1322
1323     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1324         $kvm_api_version = $v;
1325     }
1326
1327     $fh->close();
1328
1329     return  $kvm_api_version;
1330 }
1331
1332 my $kvm_user_version;
1333
1334 sub kvm_user_version {
1335
1336     return $kvm_user_version if $kvm_user_version;
1337
1338     $kvm_user_version = 'unknown';
1339
1340     my $code = sub {
1341         my $line = shift;
1342         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1343             $kvm_user_version = $2;
1344         }
1345     };
1346
1347     eval { run_command("kvm -version", outfunc => $code); };
1348     warn $@ if $@;
1349
1350     return $kvm_user_version;
1351
1352 }
1353
1354 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1355
1356 sub valid_drive_names {
1357     # order is important - used to autoselect boot disk
1358     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1359             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1360             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1361             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1362             'efidisk0');
1363 }
1364
1365 sub is_valid_drivename {
1366     my $dev = shift;
1367
1368     return defined($drivename_hash->{$dev});
1369 }
1370
1371 sub option_exists {
1372     my $key = shift;
1373     return defined($confdesc->{$key});
1374 }
1375
1376 sub nic_models {
1377     return $nic_model_list;
1378 }
1379
1380 sub os_list_description {
1381
1382     return {
1383         other => 'Other',
1384         wxp => 'Windows XP',
1385         w2k => 'Windows 2000',
1386         w2k3 =>, 'Windows 2003',
1387         w2k8 => 'Windows 2008',
1388         wvista => 'Windows Vista',
1389         win7 => 'Windows 7',
1390         win8 => 'Windows 8/2012',
1391         win10 => 'Windows 10/2016',
1392         l24 => 'Linux 2.4',
1393         l26 => 'Linux 2.6',
1394     };
1395 }
1396
1397 my $cdrom_path;
1398
1399 sub get_cdrom_path {
1400
1401     return  $cdrom_path if $cdrom_path;
1402
1403     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1404     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1405     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1406 }
1407
1408 sub get_iso_path {
1409     my ($storecfg, $vmid, $cdrom) = @_;
1410
1411     if ($cdrom eq 'cdrom') {
1412         return get_cdrom_path();
1413     } elsif ($cdrom eq 'none') {
1414         return '';
1415     } elsif ($cdrom =~ m|^/|) {
1416         return $cdrom;
1417     } else {
1418         return PVE::Storage::path($storecfg, $cdrom);
1419     }
1420 }
1421
1422 # try to convert old style file names to volume IDs
1423 sub filename_to_volume_id {
1424     my ($vmid, $file, $media) = @_;
1425
1426      if (!($file eq 'none' || $file eq 'cdrom' ||
1427           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1428
1429         return undef if $file =~ m|/|;
1430
1431         if ($media && $media eq 'cdrom') {
1432             $file = "local:iso/$file";
1433         } else {
1434             $file = "local:$vmid/$file";
1435         }
1436     }
1437
1438     return $file;
1439 }
1440
1441 sub verify_media_type {
1442     my ($opt, $vtype, $media) = @_;
1443
1444     return if !$media;
1445
1446     my $etype;
1447     if ($media eq 'disk') {
1448         $etype = 'images';
1449     } elsif ($media eq 'cdrom') {
1450         $etype = 'iso';
1451     } else {
1452         die "internal error";
1453     }
1454
1455     return if ($vtype eq $etype);
1456
1457     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1458 }
1459
1460 sub cleanup_drive_path {
1461     my ($opt, $storecfg, $drive) = @_;
1462
1463     # try to convert filesystem paths to volume IDs
1464
1465     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1466         ($drive->{file} !~ m|^/dev/.+|) &&
1467         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1468         ($drive->{file} !~ m/^\d+$/)) {
1469         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1470         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1471         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1472         verify_media_type($opt, $vtype, $drive->{media});
1473         $drive->{file} = $volid;
1474     }
1475
1476     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1477 }
1478
1479 sub parse_hotplug_features {
1480     my ($data) = @_;
1481
1482     my $res = {};
1483
1484     return $res if $data eq '0';
1485
1486     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1487
1488     foreach my $feature (PVE::Tools::split_list($data)) {
1489         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1490             $res->{$1} = 1;
1491         } else {
1492             die "invalid hotplug feature '$feature'\n";
1493         }
1494     }
1495     return $res;
1496 }
1497
1498 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1499 sub pve_verify_hotplug_features {
1500     my ($value, $noerr) = @_;
1501
1502     return $value if parse_hotplug_features($value);
1503
1504     return undef if $noerr;
1505
1506     die "unable to parse hotplug option\n";
1507 }
1508
1509 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1510 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1511 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1512 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1513 #        [,iothread=on][,serial=serial][,model=model]
1514
1515 sub parse_drive {
1516     my ($key, $data) = @_;
1517
1518     my ($interface, $index);
1519
1520     if ($key =~ m/^([^\d]+)(\d+)$/) {
1521         $interface = $1;
1522         $index = $2;
1523     } else {
1524         return undef;
1525     }
1526
1527     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1528                                      : $confdesc->{$key}->{format};
1529     if (!$desc) {
1530         warn "invalid drive key: $key\n";
1531         return undef;
1532     }
1533     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1534     return undef if !$res;
1535     $res->{interface} = $interface;
1536     $res->{index} = $index;
1537
1538     my $error = 0;
1539     foreach my $opt (qw(bps bps_rd bps_wr)) {
1540         if (my $bps = defined(delete $res->{$opt})) {
1541             if (defined($res->{"m$opt"})) {
1542                 warn "both $opt and m$opt specified\n";
1543                 ++$error;
1544                 next;
1545             }
1546             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1547         }
1548     }
1549
1550     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1551     for my $requirement (
1552         [mbps_max => 'mbps'],
1553         [mbps_rd_max => 'mbps_rd'],
1554         [mbps_wr_max => 'mbps_wr'],
1555         [miops_max => 'miops'],
1556         [miops_rd_max => 'miops_rd'],
1557         [miops_wr_max => 'miops_wr'],
1558         [bps_max_length => 'mbps_max'],
1559         [bps_rd_max_length => 'mbps_rd_max'],
1560         [bps_wr_max_length => 'mbps_wr_max'],
1561         [iops_max_length => 'iops_max'],
1562         [iops_rd_max_length => 'iops_rd_max'],
1563         [iops_wr_max_length => 'iops_wr_max']) {
1564         my ($option, $requires) = @$requirement;
1565         if ($res->{$option} && !$res->{$requires}) {
1566             warn "$option requires $requires\n";
1567             ++$error;
1568         }
1569     }
1570
1571     return undef if $error;
1572
1573     return undef if $res->{mbps_rd} && $res->{mbps};
1574     return undef if $res->{mbps_wr} && $res->{mbps};
1575     return undef if $res->{iops_rd} && $res->{iops};
1576     return undef if $res->{iops_wr} && $res->{iops};
1577
1578     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1579         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1580         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1581         return undef if $res->{interface} eq 'virtio';
1582     }
1583
1584     if (my $size = $res->{size}) {
1585         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1586     }
1587
1588     return $res;
1589 }
1590
1591 sub print_drive {
1592     my ($vmid, $drive) = @_;
1593     my $data = { %$drive };
1594     delete $data->{$_} for qw(index interface);
1595     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1596 }
1597
1598 sub scsi_inquiry {
1599     my($fh, $noerr) = @_;
1600
1601     my $SG_IO = 0x2285;
1602     my $SG_GET_VERSION_NUM = 0x2282;
1603
1604     my $versionbuf = "\x00" x 8;
1605     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1606     if (!$ret) {
1607         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1608         return undef;
1609     }
1610     my $version = unpack("I", $versionbuf);
1611     if ($version < 30000) {
1612         die "scsi generic interface too old\n"  if !$noerr;
1613         return undef;
1614     }
1615
1616     my $buf = "\x00" x 36;
1617     my $sensebuf = "\x00" x 8;
1618     my $cmd = pack("C x3 C x1", 0x12, 36);
1619
1620     # see /usr/include/scsi/sg.h
1621     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1622
1623     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1624                       length($sensebuf), 0, length($buf), $buf,
1625                       $cmd, $sensebuf, 6000);
1626
1627     $ret = ioctl($fh, $SG_IO, $packet);
1628     if (!$ret) {
1629         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1630         return undef;
1631     }
1632
1633     my @res = unpack($sg_io_hdr_t, $packet);
1634     if ($res[17] || $res[18]) {
1635         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1636         return undef;
1637     }
1638
1639     my $res = {};
1640     (my $byte0, my $byte1, $res->{vendor},
1641      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1642
1643     $res->{removable} = $byte1 & 128 ? 1 : 0;
1644     $res->{type} = $byte0 & 31;
1645
1646     return $res;
1647 }
1648
1649 sub path_is_scsi {
1650     my ($path) = @_;
1651
1652     my $fh = IO::File->new("+<$path") || return undef;
1653     my $res = scsi_inquiry($fh, 1);
1654     close($fh);
1655
1656     return $res;
1657 }
1658
1659 sub machine_type_is_q35 {
1660     my ($conf) = @_;
1661
1662     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1663 }
1664
1665 sub print_tabletdevice_full {
1666     my ($conf) = @_;
1667
1668     my $q35 = machine_type_is_q35($conf);
1669
1670     # we use uhci for old VMs because tablet driver was buggy in older qemu
1671     my $usbbus = $q35 ? "ehci" : "uhci";
1672
1673     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1674 }
1675
1676 sub print_drivedevice_full {
1677     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1678
1679     my $device = '';
1680     my $maxdev = 0;
1681
1682     if ($drive->{interface} eq 'virtio') {
1683         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1684         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1685         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1686     } elsif ($drive->{interface} eq 'scsi') {
1687
1688         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1689         my $unit = $drive->{index} % $maxdev;
1690         my $devicetype = 'hd';
1691         my $path = '';
1692         if (drive_is_cdrom($drive)) {
1693             $devicetype = 'cd';
1694         } else {
1695             if ($drive->{file} =~ m|^/|) {
1696                 $path = $drive->{file};
1697                 if (my $info = path_is_scsi($path)) {
1698                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1699                         $devicetype = 'block';
1700                     } elsif ($info->{type} == 1) { # tape
1701                         $devicetype = 'generic';
1702                     }
1703                 }
1704             } else {
1705                  $path = PVE::Storage::path($storecfg, $drive->{file});
1706             }
1707
1708             if($path =~ m/^iscsi\:\/\//){
1709                 $devicetype = 'generic';
1710             }
1711         }
1712
1713         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1714             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1715         } else {
1716             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1717         }
1718
1719         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1720             $device .= ",rotation_rate=1";
1721         }
1722
1723     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1724         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1725         my $controller = int($drive->{index} / $maxdev);
1726         my $unit = $drive->{index} % $maxdev;
1727         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1728
1729         $device = "ide-$devicetype";
1730         if ($drive->{interface} eq 'ide') {
1731             $device .= ",bus=ide.$controller,unit=$unit";
1732         } else {
1733             $device .= ",bus=ahci$controller.$unit";
1734         }
1735         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1736
1737         if ($devicetype eq 'hd') {
1738             if (my $model = $drive->{model}) {
1739                 $model = URI::Escape::uri_unescape($model);
1740                 $device .= ",model=$model";
1741             }
1742             if ($drive->{ssd}) {
1743                 $device .= ",rotation_rate=1";
1744             }
1745         }
1746     } elsif ($drive->{interface} eq 'usb') {
1747         die "implement me";
1748         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1749     } else {
1750         die "unsupported interface type";
1751     }
1752
1753     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1754
1755     if (my $serial = $drive->{serial}) {
1756         $serial = URI::Escape::uri_unescape($serial);
1757         $device .= ",serial=$serial";
1758     }
1759
1760
1761     return $device;
1762 }
1763
1764 sub get_initiator_name {
1765     my $initiator;
1766
1767     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1768     while (defined(my $line = <$fh>)) {
1769         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1770         $initiator = $1;
1771         last;
1772     }
1773     $fh->close();
1774
1775     return $initiator;
1776 }
1777
1778 sub print_drive_full {
1779     my ($storecfg, $vmid, $drive) = @_;
1780
1781     my $path;
1782     my $volid = $drive->{file};
1783     my $format;
1784
1785     if (drive_is_cdrom($drive)) {
1786         $path = get_iso_path($storecfg, $vmid, $volid);
1787     } else {
1788         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1789         if ($storeid) {
1790             $path = PVE::Storage::path($storecfg, $volid);
1791             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1792             $format = qemu_img_format($scfg, $volname);
1793         } else {
1794             $path = $volid;
1795             $format = "raw";
1796         }
1797    }
1798
1799     my $opts = '';
1800     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1801     foreach my $o (@qemu_drive_options) {
1802         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1803     }
1804
1805     # snapshot only accepts on|off
1806     if (defined($drive->{snapshot})) {
1807         my $v = $drive->{snapshot} ? 'on' : 'off';
1808         $opts .= ",snapshot=$v";
1809     }
1810
1811     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1812         my ($dir, $qmpname) = @$type;
1813         if (my $v = $drive->{"mbps$dir"}) {
1814             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1815         }
1816         if (my $v = $drive->{"mbps${dir}_max"}) {
1817             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1818         }
1819         if (my $v = $drive->{"bps${dir}_max_length"}) {
1820             $opts .= ",throttling.bps$qmpname-max-length=$v";
1821         }
1822         if (my $v = $drive->{"iops${dir}"}) {
1823             $opts .= ",throttling.iops$qmpname=$v";
1824         }
1825         if (my $v = $drive->{"iops${dir}_max"}) {
1826             $opts .= ",throttling.iops$qmpname-max=$v";
1827         }
1828         if (my $v = $drive->{"iops${dir}_max_length"}) {
1829             $opts .= ",throttling.iops$qmpname-max-length=$v";
1830         }
1831     }
1832
1833     $opts .= ",format=$format" if $format && !$drive->{format};
1834
1835     my $cache_direct = 0;
1836
1837     if (my $cache = $drive->{cache}) {
1838         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1839     } elsif (!drive_is_cdrom($drive)) {
1840         $opts .= ",cache=none";
1841         $cache_direct = 1;
1842     }
1843
1844     # aio native works only with O_DIRECT
1845     if (!$drive->{aio}) {
1846         if($cache_direct) {
1847             $opts .= ",aio=native";
1848         } else {
1849             $opts .= ",aio=threads";
1850         }
1851     }
1852
1853     if (!drive_is_cdrom($drive)) {
1854         my $detectzeroes;
1855         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1856             $detectzeroes = 'off';
1857         } elsif ($drive->{discard}) {
1858             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1859         } else {
1860             # This used to be our default with discard not being specified:
1861             $detectzeroes = 'on';
1862         }
1863         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1864     }
1865
1866     my $pathinfo = $path ? "file=$path," : '';
1867
1868     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1869 }
1870
1871 sub print_netdevice_full {
1872     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1873
1874     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1875
1876     my $device = $net->{model};
1877     if ($net->{model} eq 'virtio') {
1878          $device = 'virtio-net-pci';
1879      };
1880
1881     my $pciaddr = print_pci_addr("$netid", $bridges);
1882     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1883     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1884         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1885         my $vectors = $net->{queues} * 2 + 2;
1886         $tmpstr .= ",vectors=$vectors,mq=on";
1887     }
1888     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1889
1890     if ($use_old_bios_files) {
1891         my $romfile;
1892         if ($device eq 'virtio-net-pci') {
1893             $romfile = 'pxe-virtio.rom';
1894         } elsif ($device eq 'e1000') {
1895             $romfile = 'pxe-e1000.rom';
1896         } elsif ($device eq 'ne2k') {
1897             $romfile = 'pxe-ne2k_pci.rom';
1898         } elsif ($device eq 'pcnet') {
1899             $romfile = 'pxe-pcnet.rom';
1900         } elsif ($device eq 'rtl8139') {
1901             $romfile = 'pxe-rtl8139.rom';
1902         }
1903         $tmpstr .= ",romfile=$romfile" if $romfile;
1904     }
1905
1906     return $tmpstr;
1907 }
1908
1909 sub print_netdev_full {
1910     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1911
1912     my $i = '';
1913     if ($netid =~ m/^net(\d+)$/) {
1914         $i = int($1);
1915     }
1916
1917     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1918
1919     my $ifname = "tap${vmid}i$i";
1920
1921     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1922     die "interface name '$ifname' is too long (max 15 character)\n"
1923         if length($ifname) >= 16;
1924
1925     my $vhostparam = '';
1926     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1927
1928     my $vmname = $conf->{name} || "vm$vmid";
1929
1930     my $netdev = "";
1931     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1932
1933     if ($net->{bridge}) {
1934         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1935     } else {
1936         $netdev = "type=user,id=$netid,hostname=$vmname";
1937     }
1938
1939     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1940
1941     return $netdev;
1942 }
1943
1944
1945 sub print_cpu_device {
1946     my ($conf, $id) = @_;
1947
1948     my $kvm = $conf->{kvm} // 1;
1949     my $cpu = $kvm ? "kvm64" : "qemu64";
1950     if (my $cputype = $conf->{cpu}) {
1951         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1952             or die "Cannot parse cpu description: $cputype\n";
1953         $cpu = $cpuconf->{cputype};
1954     }
1955
1956     my $cores = $conf->{cores} || 1;
1957
1958     my $current_core = ($id - 1) % $cores;
1959     my $current_socket = int(($id - 1 - $current_core)/$cores);
1960
1961     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1962 }
1963
1964 sub drive_is_cloudinit {
1965     my ($drive) = @_;
1966     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1967 }
1968
1969 sub drive_is_cdrom {
1970     my ($drive, $exclude_cloudinit) = @_;
1971
1972     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1973
1974     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1975
1976 }
1977
1978 sub parse_number_sets {
1979     my ($set) = @_;
1980     my $res = [];
1981     foreach my $part (split(/;/, $set)) {
1982         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1983             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1984             push @$res, [ $1, $2 ];
1985         } else {
1986             die "invalid range: $part\n";
1987         }
1988     }
1989     return $res;
1990 }
1991
1992 sub parse_numa {
1993     my ($data) = @_;
1994
1995     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1996     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1997     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1998     return $res;
1999 }
2000
2001 sub parse_hostpci {
2002     my ($value) = @_;
2003
2004     return undef if !$value;
2005
2006     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2007
2008     my @idlist = split(/;/, $res->{host});
2009     delete $res->{host};
2010     foreach my $id (@idlist) {
2011         if ($id =~ /^$PCIRE$/) {
2012             if (defined($2)) {
2013                 push @{$res->{pciid}}, { id => $1, function => $2 };
2014             } else {
2015                 my $pcidevices = lspci($1);
2016                 $res->{pciid} = $pcidevices->{$1};
2017             }
2018         } else {
2019             # should have been caught by parse_property_string already
2020             die "failed to parse PCI id: $id\n";
2021         }
2022     }
2023     return $res;
2024 }
2025
2026 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2027 sub parse_net {
2028     my ($data) = @_;
2029
2030     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2031     if ($@) {
2032         warn $@;
2033         return undef;
2034     }
2035     if (!defined($res->{macaddr})) {
2036         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2037         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2038     }
2039     return $res;
2040 }
2041
2042 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2043 sub parse_ipconfig {
2044     my ($data) = @_;
2045
2046     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2047     if ($@) {
2048         warn $@;
2049         return undef;
2050     }
2051
2052     if ($res->{gw} && !$res->{ip}) {
2053         warn 'gateway specified without specifying an IP address';
2054         return undef;
2055     }
2056     if ($res->{gw6} && !$res->{ip6}) {
2057         warn 'IPv6 gateway specified without specifying an IPv6 address';
2058         return undef;
2059     }
2060     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2061         warn 'gateway specified together with DHCP';
2062         return undef;
2063     }
2064     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2065         # gw6 + auto/dhcp
2066         warn "IPv6 gateway specified together with $res->{ip6} address";
2067         return undef;
2068     }
2069
2070     if (!$res->{ip} && !$res->{ip6}) {
2071         return { ip => 'dhcp', ip6 => 'dhcp' };
2072     }
2073
2074     return $res;
2075 }
2076
2077 sub print_net {
2078     my $net = shift;
2079
2080     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2081 }
2082
2083 sub add_random_macs {
2084     my ($settings) = @_;
2085
2086     foreach my $opt (keys %$settings) {
2087         next if $opt !~ m/^net(\d+)$/;
2088         my $net = parse_net($settings->{$opt});
2089         next if !$net;
2090         $settings->{$opt} = print_net($net);
2091     }
2092 }
2093
2094 sub vm_is_volid_owner {
2095     my ($storecfg, $vmid, $volid) = @_;
2096
2097     if ($volid !~  m|^/|) {
2098         my ($path, $owner);
2099         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2100         if ($owner && ($owner == $vmid)) {
2101             return 1;
2102         }
2103     }
2104
2105     return undef;
2106 }
2107
2108 sub split_flagged_list {
2109     my $text = shift || '';
2110     $text =~ s/[,;]/ /g;
2111     $text =~ s/^\s+//;
2112     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2113 }
2114
2115 sub join_flagged_list {
2116     my ($how, $lst) = @_;
2117     join $how, map { $lst->{$_} . $_ } keys %$lst;
2118 }
2119
2120 sub vmconfig_delete_pending_option {
2121     my ($conf, $key, $force) = @_;
2122
2123     delete $conf->{pending}->{$key};
2124     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2125     $pending_delete_hash->{$key} = $force ? '!' : '';
2126     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2127 }
2128
2129 sub vmconfig_undelete_pending_option {
2130     my ($conf, $key) = @_;
2131
2132     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2133     delete $pending_delete_hash->{$key};
2134
2135     if (%$pending_delete_hash) {
2136         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2137     } else {
2138         delete $conf->{pending}->{delete};
2139     }
2140 }
2141
2142 sub vmconfig_register_unused_drive {
2143     my ($storecfg, $vmid, $conf, $drive) = @_;
2144
2145     if (drive_is_cloudinit($drive)) {
2146         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2147         warn $@ if $@;
2148     } elsif (!drive_is_cdrom($drive)) {
2149         my $volid = $drive->{file};
2150         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2151             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2152         }
2153     }
2154 }
2155
2156 sub vmconfig_cleanup_pending {
2157     my ($conf) = @_;
2158
2159     # remove pending changes when nothing changed
2160     my $changes;
2161     foreach my $opt (keys %{$conf->{pending}}) {
2162         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2163             $changes = 1;
2164             delete $conf->{pending}->{$opt};
2165         }
2166     }
2167
2168     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2169     my $pending_delete_hash = {};
2170     while (my ($opt, $force) = each %$current_delete_hash) {
2171         if (defined($conf->{$opt})) {
2172             $pending_delete_hash->{$opt} = $force;
2173         } else {
2174             $changes = 1;
2175         }
2176     }
2177
2178     if (%$pending_delete_hash) {
2179         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2180     } else {
2181         delete $conf->{pending}->{delete};
2182     }
2183
2184     return $changes;
2185 }
2186
2187 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2188 my $smbios1_fmt = {
2189     uuid => {
2190         type => 'string',
2191         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2192         format_description => 'UUID',
2193         description => "Set SMBIOS1 UUID.",
2194         optional => 1,
2195     },
2196     version => {
2197         type => 'string',
2198         pattern => '\S+',
2199         format_description => 'string',
2200         description => "Set SMBIOS1 version.",
2201         optional => 1,
2202     },
2203     serial => {
2204         type => 'string',
2205         pattern => '\S+',
2206         format_description => 'string',
2207         description => "Set SMBIOS1 serial number.",
2208         optional => 1,
2209     },
2210     manufacturer => {
2211         type => 'string',
2212         pattern => '\S+',
2213         format_description => 'string',
2214         description => "Set SMBIOS1 manufacturer.",
2215         optional => 1,
2216     },
2217     product => {
2218         type => 'string',
2219         pattern => '\S+',
2220         format_description => 'string',
2221         description => "Set SMBIOS1 product ID.",
2222         optional => 1,
2223     },
2224     sku => {
2225         type => 'string',
2226         pattern => '\S+',
2227         format_description => 'string',
2228         description => "Set SMBIOS1 SKU string.",
2229         optional => 1,
2230     },
2231     family => {
2232         type => 'string',
2233         pattern => '\S+',
2234         format_description => 'string',
2235         description => "Set SMBIOS1 family string.",
2236         optional => 1,
2237     },
2238 };
2239
2240 sub parse_smbios1 {
2241     my ($data) = @_;
2242
2243     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2244     warn $@ if $@;
2245     return $res;
2246 }
2247
2248 sub print_smbios1 {
2249     my ($smbios1) = @_;
2250     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2251 }
2252
2253 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2254
2255 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2256 sub verify_bootdisk {
2257     my ($value, $noerr) = @_;
2258
2259     return $value if is_valid_drivename($value);
2260
2261     return undef if $noerr;
2262
2263     die "invalid boot disk '$value'\n";
2264 }
2265
2266 sub parse_watchdog {
2267     my ($value) = @_;
2268
2269     return undef if !$value;
2270
2271     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2272     warn $@ if $@;
2273     return $res;
2274 }
2275
2276 sub parse_guest_agent {
2277     my ($value) = @_;
2278
2279     return {} if !defined($value->{agent});
2280
2281     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2282     warn $@ if $@;
2283
2284     # if the agent is disabled ignore the other potentially set properties
2285     return {} if !$res->{enabled};
2286     return $res;
2287 }
2288
2289 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2290 sub verify_usb_device {
2291     my ($value, $noerr) = @_;
2292
2293     return $value if parse_usb_device($value);
2294
2295     return undef if $noerr;
2296
2297     die "unable to parse usb device\n";
2298 }
2299
2300 # add JSON properties for create and set function
2301 sub json_config_properties {
2302     my $prop = shift;
2303
2304     foreach my $opt (keys %$confdesc) {
2305         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2306         $prop->{$opt} = $confdesc->{$opt};
2307     }
2308
2309     return $prop;
2310 }
2311
2312 # return copy of $confdesc_cloudinit to generate documentation
2313 sub cloudinit_config_properties {
2314
2315     return dclone($confdesc_cloudinit);
2316 }
2317
2318 sub check_type {
2319     my ($key, $value) = @_;
2320
2321     die "unknown setting '$key'\n" if !$confdesc->{$key};
2322
2323     my $type = $confdesc->{$key}->{type};
2324
2325     if (!defined($value)) {
2326         die "got undefined value\n";
2327     }
2328
2329     if ($value =~ m/[\n\r]/) {
2330         die "property contains a line feed\n";
2331     }
2332
2333     if ($type eq 'boolean') {
2334         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2335         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2336         die "type check ('boolean') failed - got '$value'\n";
2337     } elsif ($type eq 'integer') {
2338         return int($1) if $value =~ m/^(\d+)$/;
2339         die "type check ('integer') failed - got '$value'\n";
2340     } elsif ($type eq 'number') {
2341         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2342         die "type check ('number') failed - got '$value'\n";
2343     } elsif ($type eq 'string') {
2344         if (my $fmt = $confdesc->{$key}->{format}) {
2345             PVE::JSONSchema::check_format($fmt, $value);
2346             return $value;
2347         }
2348         $value =~ s/^\"(.*)\"$/$1/;
2349         return $value;
2350     } else {
2351         die "internal error"
2352     }
2353 }
2354
2355 sub check_iommu_support{
2356     #fixme : need to check IOMMU support
2357     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2358
2359     my $iommu=1;
2360     return $iommu;
2361
2362 }
2363
2364 sub touch_config {
2365     my ($vmid) = @_;
2366
2367     my $conf = PVE::QemuConfig->config_file($vmid);
2368     utime undef, undef, $conf;
2369 }
2370
2371 sub destroy_vm {
2372     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2373
2374     my $conffile = PVE::QemuConfig->config_file($vmid);
2375
2376     my $conf = PVE::QemuConfig->load_config($vmid);
2377
2378     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2379
2380     if ($conf->{template}) {
2381         # check if any base image is still used by a linked clone
2382         foreach_drive($conf, sub {
2383                 my ($ds, $drive) = @_;
2384
2385                 return if drive_is_cdrom($drive);
2386
2387                 my $volid = $drive->{file};
2388
2389                 return if !$volid || $volid =~ m|^/|;
2390
2391                 die "base volume '$volid' is still in use by linked cloned\n"
2392                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2393
2394         });
2395     }
2396
2397     # only remove disks owned by this VM
2398     foreach_drive($conf, sub {
2399         my ($ds, $drive) = @_;
2400
2401         return if drive_is_cdrom($drive, 1);
2402
2403         my $volid = $drive->{file};
2404
2405         return if !$volid || $volid =~ m|^/|;
2406
2407         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2408         return if !$path || !$owner || ($owner != $vmid);
2409
2410         eval {
2411             PVE::Storage::vdisk_free($storecfg, $volid);
2412         };
2413         warn "Could not remove disk '$volid', check manually: $@" if $@;
2414
2415     });
2416
2417     if ($keep_empty_config) {
2418         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2419     } else {
2420         unlink $conffile;
2421     }
2422
2423     # also remove unused disk
2424     eval {
2425         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2426
2427         eval {
2428             PVE::Storage::foreach_volid($dl, sub {
2429                 my ($volid, $sid, $volname, $d) = @_;
2430                 PVE::Storage::vdisk_free($storecfg, $volid);
2431             });
2432         };
2433         warn $@ if $@;
2434
2435     };
2436     warn $@ if $@;
2437 }
2438
2439 sub parse_vm_config {
2440     my ($filename, $raw) = @_;
2441
2442     return undef if !defined($raw);
2443
2444     my $res = {
2445         digest => Digest::SHA::sha1_hex($raw),
2446         snapshots => {},
2447         pending => {},
2448     };
2449
2450     $filename =~ m|/qemu-server/(\d+)\.conf$|
2451         || die "got strange filename '$filename'";
2452
2453     my $vmid = $1;
2454
2455     my $conf = $res;
2456     my $descr;
2457     my $section = '';
2458
2459     my @lines = split(/\n/, $raw);
2460     foreach my $line (@lines) {
2461         next if $line =~ m/^\s*$/;
2462
2463         if ($line =~ m/^\[PENDING\]\s*$/i) {
2464             $section = 'pending';
2465             if (defined($descr)) {
2466                 $descr =~ s/\s+$//;
2467                 $conf->{description} = $descr;
2468             }
2469             $descr = undef;
2470             $conf = $res->{$section} = {};
2471             next;
2472
2473         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2474             $section = $1;
2475             if (defined($descr)) {
2476                 $descr =~ s/\s+$//;
2477                 $conf->{description} = $descr;
2478             }
2479             $descr = undef;
2480             $conf = $res->{snapshots}->{$section} = {};
2481             next;
2482         }
2483
2484         if ($line =~ m/^\#(.*)\s*$/) {
2485             $descr = '' if !defined($descr);
2486             $descr .= PVE::Tools::decode_text($1) . "\n";
2487             next;
2488         }
2489
2490         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2491             $descr = '' if !defined($descr);
2492             $descr .= PVE::Tools::decode_text($2);
2493         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2494             $conf->{snapstate} = $1;
2495         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2496             my $key = $1;
2497             my $value = $2;
2498             $conf->{$key} = $value;
2499         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2500             my $value = $1;
2501             if ($section eq 'pending') {
2502                 $conf->{delete} = $value; # we parse this later
2503             } else {
2504                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2505             }
2506         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2507             my $key = $1;
2508             my $value = $2;
2509             eval { $value = check_type($key, $value); };
2510             if ($@) {
2511                 warn "vm $vmid - unable to parse value of '$key' - $@";
2512             } else {
2513                 $key = 'ide2' if $key eq 'cdrom';
2514                 my $fmt = $confdesc->{$key}->{format};
2515                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2516                     my $v = parse_drive($key, $value);
2517                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2518                         $v->{file} = $volid;
2519                         $value = print_drive($vmid, $v);
2520                     } else {
2521                         warn "vm $vmid - unable to parse value of '$key'\n";
2522                         next;
2523                     }
2524                 }
2525
2526                 $conf->{$key} = $value;
2527             }
2528         }
2529     }
2530
2531     if (defined($descr)) {
2532         $descr =~ s/\s+$//;
2533         $conf->{description} = $descr;
2534     }
2535     delete $res->{snapstate}; # just to be sure
2536
2537     return $res;
2538 }
2539
2540 sub write_vm_config {
2541     my ($filename, $conf) = @_;
2542
2543     delete $conf->{snapstate}; # just to be sure
2544
2545     if ($conf->{cdrom}) {
2546         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2547         $conf->{ide2} = $conf->{cdrom};
2548         delete $conf->{cdrom};
2549     }
2550
2551     # we do not use 'smp' any longer
2552     if ($conf->{sockets}) {
2553         delete $conf->{smp};
2554     } elsif ($conf->{smp}) {
2555         $conf->{sockets} = $conf->{smp};
2556         delete $conf->{cores};
2557         delete $conf->{smp};
2558     }
2559
2560     my $used_volids = {};
2561
2562     my $cleanup_config = sub {
2563         my ($cref, $pending, $snapname) = @_;
2564
2565         foreach my $key (keys %$cref) {
2566             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2567                 $key eq 'snapstate' || $key eq 'pending';
2568             my $value = $cref->{$key};
2569             if ($key eq 'delete') {
2570                 die "propertry 'delete' is only allowed in [PENDING]\n"
2571                     if !$pending;
2572                 # fixme: check syntax?
2573                 next;
2574             }
2575             eval { $value = check_type($key, $value); };
2576             die "unable to parse value of '$key' - $@" if $@;
2577
2578             $cref->{$key} = $value;
2579
2580             if (!$snapname && is_valid_drivename($key)) {
2581                 my $drive = parse_drive($key, $value);
2582                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2583             }
2584         }
2585     };
2586
2587     &$cleanup_config($conf);
2588
2589     &$cleanup_config($conf->{pending}, 1);
2590
2591     foreach my $snapname (keys %{$conf->{snapshots}}) {
2592         die "internal error" if $snapname eq 'pending';
2593         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2594     }
2595
2596     # remove 'unusedX' settings if we re-add a volume
2597     foreach my $key (keys %$conf) {
2598         my $value = $conf->{$key};
2599         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2600             delete $conf->{$key};
2601         }
2602     }
2603
2604     my $generate_raw_config = sub {
2605         my ($conf, $pending) = @_;
2606
2607         my $raw = '';
2608
2609         # add description as comment to top of file
2610         if (defined(my $descr = $conf->{description})) {
2611             if ($descr) {
2612                 foreach my $cl (split(/\n/, $descr)) {
2613                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2614                 }
2615             } else {
2616                 $raw .= "#\n" if $pending;
2617             }
2618         }
2619
2620         foreach my $key (sort keys %$conf) {
2621             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2622             $raw .= "$key: $conf->{$key}\n";
2623         }
2624         return $raw;
2625     };
2626
2627     my $raw = &$generate_raw_config($conf);
2628
2629     if (scalar(keys %{$conf->{pending}})){
2630         $raw .= "\n[PENDING]\n";
2631         $raw .= &$generate_raw_config($conf->{pending}, 1);
2632     }
2633
2634     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2635         $raw .= "\n[$snapname]\n";
2636         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2637     }
2638
2639     return $raw;
2640 }
2641
2642 sub load_defaults {
2643
2644     my $res = {};
2645
2646     # we use static defaults from our JSON schema configuration
2647     foreach my $key (keys %$confdesc) {
2648         if (defined(my $default = $confdesc->{$key}->{default})) {
2649             $res->{$key} = $default;
2650         }
2651     }
2652
2653     return $res;
2654 }
2655
2656 sub config_list {
2657     my $vmlist = PVE::Cluster::get_vmlist();
2658     my $res = {};
2659     return $res if !$vmlist || !$vmlist->{ids};
2660     my $ids = $vmlist->{ids};
2661
2662     foreach my $vmid (keys %$ids) {
2663         my $d = $ids->{$vmid};
2664         next if !$d->{node} || $d->{node} ne $nodename;
2665         next if !$d->{type} || $d->{type} ne 'qemu';
2666         $res->{$vmid}->{exists} = 1;
2667     }
2668     return $res;
2669 }
2670
2671 # test if VM uses local resources (to prevent migration)
2672 sub check_local_resources {
2673     my ($conf, $noerr) = @_;
2674
2675     my $loc_res = 0;
2676
2677     $loc_res = 1 if $conf->{hostusb}; # old syntax
2678     $loc_res = 1 if $conf->{hostpci}; # old syntax
2679
2680     foreach my $k (keys %$conf) {
2681         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2682         # sockets are safe: they will recreated be on the target side post-migrate
2683         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2684         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2685     }
2686
2687     die "VM uses local resources\n" if $loc_res && !$noerr;
2688
2689     return $loc_res;
2690 }
2691
2692 # check if used storages are available on all nodes (use by migrate)
2693 sub check_storage_availability {
2694     my ($storecfg, $conf, $node) = @_;
2695
2696     foreach_drive($conf, sub {
2697         my ($ds, $drive) = @_;
2698
2699         my $volid = $drive->{file};
2700         return if !$volid;
2701
2702         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2703         return if !$sid;
2704
2705         # check if storage is available on both nodes
2706         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2707         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2708    });
2709 }
2710
2711 # list nodes where all VM images are available (used by has_feature API)
2712 sub shared_nodes {
2713     my ($conf, $storecfg) = @_;
2714
2715     my $nodelist = PVE::Cluster::get_nodelist();
2716     my $nodehash = { map { $_ => 1 } @$nodelist };
2717     my $nodename = PVE::INotify::nodename();
2718
2719     foreach_drive($conf, sub {
2720         my ($ds, $drive) = @_;
2721
2722         my $volid = $drive->{file};
2723         return if !$volid;
2724
2725         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2726         if ($storeid) {
2727             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2728             if ($scfg->{disable}) {
2729                 $nodehash = {};
2730             } elsif (my $avail = $scfg->{nodes}) {
2731                 foreach my $node (keys %$nodehash) {
2732                     delete $nodehash->{$node} if !$avail->{$node};
2733                 }
2734             } elsif (!$scfg->{shared}) {
2735                 foreach my $node (keys %$nodehash) {
2736                     delete $nodehash->{$node} if $node ne $nodename
2737                 }
2738             }
2739         }
2740     });
2741
2742     return $nodehash
2743 }
2744
2745 sub check_cmdline {
2746     my ($pidfile, $pid) = @_;
2747
2748     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2749     if (defined($fh)) {
2750         my $line = <$fh>;
2751         $fh->close;
2752         return undef if !$line;
2753         my @param = split(/\0/, $line);
2754
2755         my $cmd = $param[0];
2756         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2757
2758         for (my $i = 0; $i < scalar (@param); $i++) {
2759             my $p = $param[$i];
2760             next if !$p;
2761             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2762                 my $p = $param[$i+1];
2763                 return 1 if $p && ($p eq $pidfile);
2764                 return undef;
2765             }
2766         }
2767     }
2768     return undef;
2769 }
2770
2771 sub check_running {
2772     my ($vmid, $nocheck, $node) = @_;
2773
2774     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2775
2776     die "unable to find configuration file for VM $vmid - no such machine\n"
2777         if !$nocheck && ! -f $filename;
2778
2779     my $pidfile = pidfile_name($vmid);
2780
2781     if (my $fd = IO::File->new("<$pidfile")) {
2782         my $st = stat($fd);
2783         my $line = <$fd>;
2784         close($fd);
2785
2786         my $mtime = $st->mtime;
2787         if ($mtime > time()) {
2788             warn "file '$filename' modified in future\n";
2789         }
2790
2791         if ($line =~ m/^(\d+)$/) {
2792             my $pid = $1;
2793             if (check_cmdline($pidfile, $pid)) {
2794                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2795                     return $pid;
2796                 }
2797             }
2798         }
2799     }
2800
2801     return undef;
2802 }
2803
2804 sub vzlist {
2805
2806     my $vzlist = config_list();
2807
2808     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2809
2810     while (defined(my $de = $fd->read)) {
2811         next if $de !~ m/^(\d+)\.pid$/;
2812         my $vmid = $1;
2813         next if !defined($vzlist->{$vmid});
2814         if (my $pid = check_running($vmid)) {
2815             $vzlist->{$vmid}->{pid} = $pid;
2816         }
2817     }
2818
2819     return $vzlist;
2820 }
2821
2822 sub disksize {
2823     my ($storecfg, $conf) = @_;
2824
2825     my $bootdisk = $conf->{bootdisk};
2826     return undef if !$bootdisk;
2827     return undef if !is_valid_drivename($bootdisk);
2828
2829     return undef if !$conf->{$bootdisk};
2830
2831     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2832     return undef if !defined($drive);
2833
2834     return undef if drive_is_cdrom($drive);
2835
2836     my $volid = $drive->{file};
2837     return undef if !$volid;
2838
2839     return $drive->{size};
2840 }
2841
2842 our $vmstatus_return_properties = {
2843     vmid => get_standard_option('pve-vmid'),
2844     status => {
2845         description => "Qemu process status.",
2846         type => 'string',
2847         enum => ['stopped', 'running'],
2848     },
2849     maxmem => {
2850         description => "Maximum memory in bytes.",
2851         type => 'integer',
2852         optional => 1,
2853         renderer => 'bytes',
2854     },
2855     maxdisk => {
2856         description => "Root disk size in bytes.",
2857         type => 'integer',
2858         optional => 1,
2859         renderer => 'bytes',
2860     },
2861     name => {
2862         description => "VM name.",
2863         type => 'string',
2864         optional => 1,
2865     },
2866     qmpstatus => {
2867         description => "Qemu QMP agent status.",
2868         type => 'string',
2869         optional => 1,
2870     },
2871     pid => {
2872         description => "PID of running qemu process.",
2873         type => 'integer',
2874         optional => 1,
2875     },
2876     uptime => {
2877         description => "Uptime.",
2878         type => 'integer',
2879         optional => 1,
2880         renderer => 'duration',
2881     },
2882     cpus => {
2883         description => "Maximum usable CPUs.",
2884         type => 'number',
2885         optional => 1,
2886     },
2887 };
2888
2889 my $last_proc_pid_stat;
2890
2891 # get VM status information
2892 # This must be fast and should not block ($full == false)
2893 # We only query KVM using QMP if $full == true (this can be slow)
2894 sub vmstatus {
2895     my ($opt_vmid, $full) = @_;
2896
2897     my $res = {};
2898
2899     my $storecfg = PVE::Storage::config();
2900
2901     my $list = vzlist();
2902     my $defaults = load_defaults();
2903
2904     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2905
2906     my $cpucount = $cpuinfo->{cpus} || 1;
2907
2908     foreach my $vmid (keys %$list) {
2909         next if $opt_vmid && ($vmid ne $opt_vmid);
2910
2911         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2912         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2913
2914         my $d = { vmid => $vmid };
2915         $d->{pid} = $list->{$vmid}->{pid};
2916
2917         # fixme: better status?
2918         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2919
2920         my $size = disksize($storecfg, $conf);
2921         if (defined($size)) {
2922             $d->{disk} = 0; # no info available
2923             $d->{maxdisk} = $size;
2924         } else {
2925             $d->{disk} = 0;
2926             $d->{maxdisk} = 0;
2927         }
2928
2929         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2930             * ($conf->{cores} || $defaults->{cores});
2931         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2932         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2933
2934         $d->{name} = $conf->{name} || "VM $vmid";
2935         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2936             : $defaults->{memory}*(1024*1024);
2937
2938         if ($conf->{balloon}) {
2939             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2940             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2941                 : $defaults->{shares};
2942         }
2943
2944         $d->{uptime} = 0;
2945         $d->{cpu} = 0;
2946         $d->{mem} = 0;
2947
2948         $d->{netout} = 0;
2949         $d->{netin} = 0;
2950
2951         $d->{diskread} = 0;
2952         $d->{diskwrite} = 0;
2953
2954         $d->{template} = PVE::QemuConfig->is_template($conf);
2955
2956         $d->{serial} = 1 if conf_has_serial($conf);
2957
2958         $res->{$vmid} = $d;
2959     }
2960
2961     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2962     foreach my $dev (keys %$netdev) {
2963         next if $dev !~ m/^tap([1-9]\d*)i/;
2964         my $vmid = $1;
2965         my $d = $res->{$vmid};
2966         next if !$d;
2967
2968         $d->{netout} += $netdev->{$dev}->{receive};
2969         $d->{netin} += $netdev->{$dev}->{transmit};
2970
2971         if ($full) {
2972             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2973             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2974         }
2975
2976     }
2977
2978     my $ctime = gettimeofday;
2979
2980     foreach my $vmid (keys %$list) {
2981
2982         my $d = $res->{$vmid};
2983         my $pid = $d->{pid};
2984         next if !$pid;
2985
2986         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2987         next if !$pstat; # not running
2988
2989         my $used = $pstat->{utime} + $pstat->{stime};
2990
2991         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2992
2993         if ($pstat->{vsize}) {
2994             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2995         }
2996
2997         my $old = $last_proc_pid_stat->{$pid};
2998         if (!$old) {
2999             $last_proc_pid_stat->{$pid} = {
3000                 time => $ctime,
3001                 used => $used,
3002                 cpu => 0,
3003             };
3004             next;
3005         }
3006
3007         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3008
3009         if ($dtime > 1000) {
3010             my $dutime = $used -  $old->{used};
3011
3012             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3013             $last_proc_pid_stat->{$pid} = {
3014                 time => $ctime,
3015                 used => $used,
3016                 cpu => $d->{cpu},
3017             };
3018         } else {
3019             $d->{cpu} = $old->{cpu};
3020         }
3021     }
3022
3023     return $res if !$full;
3024
3025     my $qmpclient = PVE::QMPClient->new();
3026
3027     my $ballooncb = sub {
3028         my ($vmid, $resp) = @_;
3029
3030         my $info = $resp->{'return'};
3031         return if !$info->{max_mem};
3032
3033         my $d = $res->{$vmid};
3034
3035         # use memory assigned to VM
3036         $d->{maxmem} = $info->{max_mem};
3037         $d->{balloon} = $info->{actual};
3038
3039         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3040             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3041             $d->{freemem} = $info->{free_mem};
3042         }
3043
3044         $d->{ballooninfo} = $info;
3045     };
3046
3047     my $blockstatscb = sub {
3048         my ($vmid, $resp) = @_;
3049         my $data = $resp->{'return'} || [];
3050         my $totalrdbytes = 0;
3051         my $totalwrbytes = 0;
3052
3053         for my $blockstat (@$data) {
3054             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3055             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3056
3057             $blockstat->{device} =~ s/drive-//;
3058             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3059         }
3060         $res->{$vmid}->{diskread} = $totalrdbytes;
3061         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3062     };
3063
3064     my $statuscb = sub {
3065         my ($vmid, $resp) = @_;
3066
3067         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3068         # this fails if ballon driver is not loaded, so this must be
3069         # the last commnand (following command are aborted if this fails).
3070         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3071
3072         my $status = 'unknown';
3073         if (!defined($status = $resp->{'return'}->{status})) {
3074             warn "unable to get VM status\n";
3075             return;
3076         }
3077
3078         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3079     };
3080
3081     foreach my $vmid (keys %$list) {
3082         next if $opt_vmid && ($vmid ne $opt_vmid);
3083         next if !$res->{$vmid}->{pid}; # not running
3084         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3085     }
3086
3087     $qmpclient->queue_execute(undef, 2);
3088
3089     foreach my $vmid (keys %$list) {
3090         next if $opt_vmid && ($vmid ne $opt_vmid);
3091         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3092     }
3093
3094     return $res;
3095 }
3096
3097 sub foreach_drive {
3098     my ($conf, $func, @param) = @_;
3099
3100     foreach my $ds (valid_drive_names()) {
3101         next if !defined($conf->{$ds});
3102
3103         my $drive = parse_drive($ds, $conf->{$ds});
3104         next if !$drive;
3105
3106         &$func($ds, $drive, @param);
3107     }
3108 }
3109
3110 sub foreach_volid {
3111     my ($conf, $func, @param) = @_;
3112
3113     my $volhash = {};
3114
3115     my $test_volid = sub {
3116         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3117
3118         return if !$volid;
3119
3120         $volhash->{$volid}->{cdrom} //= 1;
3121         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3122
3123         $volhash->{$volid}->{replicate} //= 0;
3124         $volhash->{$volid}->{replicate} = 1 if $replicate;
3125
3126         $volhash->{$volid}->{shared} //= 0;
3127         $volhash->{$volid}->{shared} = 1 if $shared;
3128
3129         $volhash->{$volid}->{referenced_in_config} //= 0;
3130         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3131
3132         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3133             if defined($snapname);
3134     };
3135
3136     foreach_drive($conf, sub {
3137         my ($ds, $drive) = @_;
3138         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3139     });
3140
3141     foreach my $snapname (keys %{$conf->{snapshots}}) {
3142         my $snap = $conf->{snapshots}->{$snapname};
3143         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3144         foreach_drive($snap, sub {
3145             my ($ds, $drive) = @_;
3146             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3147         });
3148     }
3149
3150     foreach my $volid (keys %$volhash) {
3151         &$func($volid, $volhash->{$volid}, @param);
3152     }
3153 }
3154
3155 sub conf_has_serial {
3156     my ($conf) = @_;
3157
3158     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3159         if ($conf->{"serial$i"}) {
3160             return 1;
3161         }
3162     }
3163
3164     return 0;
3165 }
3166
3167 sub vga_conf_has_spice {
3168     my ($vga) = @_;
3169
3170     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3171
3172     return $1 || 1;
3173 }
3174
3175 sub config_to_command {
3176     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3177
3178     my $cmd = [];
3179     my $globalFlags = [];
3180     my $machineFlags = [];
3181     my $rtcFlags = [];
3182     my $cpuFlags = [];
3183     my $devices = [];
3184     my $pciaddr = '';
3185     my $bridges = {};
3186     my $kvmver = kvm_user_version();
3187     my $vernum = 0; # unknown
3188     my $ostype = $conf->{ostype};
3189     my $winversion = windows_version($ostype);
3190     my $kvm = $conf->{kvm} // 1;
3191
3192     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3193
3194     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3195         $vernum = $1*1000000+$2*1000;
3196     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3197         $vernum = $1*1000000+$2*1000+$3;
3198     }
3199
3200     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3201
3202     my $have_ovz = -f '/proc/vz/vestat';
3203
3204     my $q35 = machine_type_is_q35($conf);
3205     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3206     my $machine_type = $forcemachine || $conf->{machine};
3207     my $use_old_bios_files = undef;
3208     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3209
3210     my $cpuunits = defined($conf->{cpuunits}) ?
3211             $conf->{cpuunits} : $defaults->{cpuunits};
3212
3213     push @$cmd, '/usr/bin/kvm';
3214
3215     push @$cmd, '-id', $vmid;
3216
3217     my $vmname = $conf->{name} || "vm$vmid";
3218
3219     push @$cmd, '-name', $vmname;
3220
3221     my $use_virtio = 0;
3222
3223     my $qmpsocket = qmp_socket($vmid);
3224     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3225     push @$cmd, '-mon', "chardev=qmp,mode=control";
3226
3227     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3228         my $eventsocket = qmp_socket($vmid, 0, 'event');
3229         push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
3230         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3231     }
3232
3233     push @$cmd, '-pidfile' , pidfile_name($vmid);
3234
3235     push @$cmd, '-daemonize';
3236
3237     if ($conf->{smbios1}) {
3238         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3239     }
3240
3241     if ($conf->{vmgenid}) {
3242         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3243     }
3244
3245     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3246         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3247
3248         my $path;
3249         my $format;
3250         if (my $efidisk = $conf->{efidisk0}) {
3251             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3252             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3253             $format = $d->{format};
3254             if ($storeid) {
3255                 $path = PVE::Storage::path($storecfg, $d->{file});
3256                 if (!defined($format)) {
3257                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3258                     $format = qemu_img_format($scfg, $volname);
3259                 }
3260             } else {
3261                 $path = $d->{file};
3262                 die "efidisk format must be specified\n"
3263                     if !defined($format);
3264             }
3265         } else {
3266             warn "no efidisk configured! Using temporary efivars disk.\n";
3267             $path = "/tmp/$vmid-ovmf.fd";
3268             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3269             $format = 'raw';
3270         }
3271
3272         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3273         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3274     }
3275
3276
3277     # add usb controllers
3278     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3279     push @$devices, @usbcontrollers if @usbcontrollers;
3280     my $vga = $conf->{vga};
3281
3282     my $qxlnum = vga_conf_has_spice($vga);
3283     $vga = 'qxl' if $qxlnum;
3284
3285     if (!$vga) {
3286         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3287             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3288         } else {
3289             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3290         }
3291     }
3292
3293     # enable absolute mouse coordinates (needed by vnc)
3294     my $tablet;
3295     if (defined($conf->{tablet})) {
3296         $tablet = $conf->{tablet};
3297     } else {
3298         $tablet = $defaults->{tablet};
3299         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3300         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3301     }
3302
3303     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3304
3305     my $kvm_off = 0;
3306     my $gpu_passthrough;
3307
3308     # host pci devices
3309     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3310         my $d = parse_hostpci($conf->{"hostpci$i"});
3311         next if !$d;
3312
3313         my $pcie = $d->{pcie};
3314         if($pcie){
3315             die "q35 machine model is not enabled" if !$q35;
3316             $pciaddr = print_pcie_addr("hostpci$i");
3317         }else{
3318             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3319         }
3320
3321         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3322         my $romfile = $d->{romfile};
3323
3324         my $xvga = '';
3325         if ($d->{'x-vga'}) {
3326             $xvga = ',x-vga=on';
3327             $kvm_off = 1;
3328             $vga = 'none';
3329             $gpu_passthrough = 1;
3330
3331             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3332                 $xvga = "";
3333             }
3334         }
3335         my $pcidevices = $d->{pciid};
3336         my $multifunction = 1 if @$pcidevices > 1;
3337
3338         my $j=0;
3339         foreach my $pcidevice (@$pcidevices) {
3340
3341             my $id = "hostpci$i";
3342             $id .= ".$j" if $multifunction;
3343             my $addr = $pciaddr;
3344             $addr .= ".$j" if $multifunction;
3345             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3346
3347             if($j == 0){
3348                 $devicestr .= "$rombar$xvga";
3349                 $devicestr .= ",multifunction=on" if $multifunction;
3350                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3351             }
3352
3353             push @$devices, '-device', $devicestr;
3354             $j++;
3355         }
3356     }
3357
3358     # usb devices
3359     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3360     push @$devices, @usbdevices if @usbdevices;
3361     # serial devices
3362     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3363         if (my $path = $conf->{"serial$i"}) {
3364             if ($path eq 'socket') {
3365                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3366                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3367                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3368             } else {
3369                 die "no such serial device\n" if ! -c $path;
3370                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3371                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3372             }
3373         }
3374     }
3375
3376     # parallel devices
3377     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3378         if (my $path = $conf->{"parallel$i"}) {
3379             die "no such parallel device\n" if ! -c $path;
3380             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3381             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3382             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3383         }
3384     }
3385
3386
3387     my $sockets = 1;
3388     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3389     $sockets = $conf->{sockets} if  $conf->{sockets};
3390
3391     my $cores = $conf->{cores} || 1;
3392
3393     my $maxcpus = $sockets * $cores;
3394
3395     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3396
3397     my $allowed_vcpus = $cpuinfo->{cpus};
3398
3399     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3400         if ($allowed_vcpus < $maxcpus);
3401
3402     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3403
3404         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3405         for (my $i = 2; $i <= $vcpus; $i++)  {
3406             my $cpustr = print_cpu_device($conf,$i);
3407             push @$cmd, '-device', $cpustr;
3408         }
3409
3410     } else {
3411
3412         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3413     }
3414     push @$cmd, '-nodefaults';
3415
3416     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3417
3418     my $bootindex_hash = {};
3419     my $i = 1;
3420     foreach my $o (split(//, $bootorder)) {
3421         $bootindex_hash->{$o} = $i*100;
3422         $i++;
3423     }
3424
3425     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3426
3427     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3428
3429     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3430
3431     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3432
3433     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3434         my $socket = vnc_socket($vmid);
3435         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3436     } else {
3437         push @$cmd, '-nographic';
3438     }
3439
3440     # time drift fix
3441     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3442
3443     my $useLocaltime = $conf->{localtime};
3444
3445     if ($winversion >= 5) { # windows
3446         $useLocaltime = 1 if !defined($conf->{localtime});
3447
3448         # use time drift fix when acpi is enabled
3449         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3450             $tdf = 1 if !defined($conf->{tdf});
3451         }
3452     }
3453
3454     if ($winversion >= 6) {
3455         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3456         push @$cmd, '-no-hpet';
3457     }
3458
3459     push @$rtcFlags, 'driftfix=slew' if $tdf;
3460
3461     if (!$kvm) {
3462         push @$machineFlags, 'accel=tcg';
3463     }
3464
3465     if ($machine_type) {
3466         push @$machineFlags, "type=${machine_type}";
3467     }
3468
3469     if ($conf->{startdate}) {
3470         push @$rtcFlags, "base=$conf->{startdate}";
3471     } elsif ($useLocaltime) {
3472         push @$rtcFlags, 'base=localtime';
3473     }
3474
3475     my $cpu = $kvm ? "kvm64" : "qemu64";
3476     if (my $cputype = $conf->{cpu}) {
3477         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3478             or die "Cannot parse cpu description: $cputype\n";
3479         $cpu = $cpuconf->{cputype};
3480         $kvm_off = 1 if $cpuconf->{hidden};
3481
3482         if (defined(my $flags = $cpuconf->{flags})) {
3483             push @$cpuFlags, split(";", $flags);
3484         }
3485     }
3486
3487     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3488
3489     push @$cpuFlags , '-x2apic'
3490         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3491
3492     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3493
3494     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3495
3496     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3497
3498         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3499         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3500     }
3501
3502     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3503
3504     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3505
3506     push @$cpuFlags, 'kvm=off' if $kvm_off;
3507
3508     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3509         die "internal error"; # should not happen
3510
3511     push @$cpuFlags, "vendor=${cpu_vendor}"
3512         if $cpu_vendor ne 'default';
3513
3514     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3515
3516     push @$cmd, '-cpu', $cpu;
3517
3518     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3519
3520     push @$cmd, '-S' if $conf->{freeze};
3521
3522     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3523
3524     # enable sound
3525     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3526     #push @$cmd, '-soundhw', 'es1370';
3527     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3528
3529     if (parse_guest_agent($conf)->{enabled}) {
3530         my $qgasocket = qmp_socket($vmid, 1);
3531         my $pciaddr = print_pci_addr("qga0", $bridges);
3532         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3533         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3534         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3535     }
3536
3537     my $spice_port;
3538
3539     if ($qxlnum) {
3540         if ($qxlnum > 1) {
3541             if ($winversion){
3542                 for(my $i = 1; $i < $qxlnum; $i++){
3543                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3544                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3545                 }
3546             } else {
3547                 # assume other OS works like Linux
3548                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3549                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3550             }
3551         }
3552
3553         my $pciaddr = print_pci_addr("spice", $bridges);
3554
3555         my $nodename = PVE::INotify::nodename();
3556         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3557         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3558         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3559         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3560         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3561
3562         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3563
3564         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3565         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3566         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3567     }
3568
3569     # enable balloon by default, unless explicitly disabled
3570     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3571         $pciaddr = print_pci_addr("balloon0", $bridges);
3572         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3573     }
3574
3575     if ($conf->{watchdog}) {
3576         my $wdopts = parse_watchdog($conf->{watchdog});
3577         $pciaddr = print_pci_addr("watchdog", $bridges);
3578         my $watchdog = $wdopts->{model} || 'i6300esb';
3579         push @$devices, '-device', "$watchdog$pciaddr";
3580         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3581     }
3582
3583     my $vollist = [];
3584     my $scsicontroller = {};
3585     my $ahcicontroller = {};
3586     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3587
3588     # Add iscsi initiator name if available
3589     if (my $initiator = get_initiator_name()) {
3590         push @$devices, '-iscsi', "initiator-name=$initiator";
3591     }
3592
3593     foreach_drive($conf, sub {
3594         my ($ds, $drive) = @_;
3595
3596         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3597             push @$vollist, $drive->{file};
3598         }
3599
3600         # ignore efidisk here, already added in bios/fw handling code above
3601         return if $drive->{interface} eq 'efidisk';
3602
3603         $use_virtio = 1 if $ds =~ m/^virtio/;
3604
3605         if (drive_is_cdrom ($drive)) {
3606             if ($bootindex_hash->{d}) {
3607                 $drive->{bootindex} = $bootindex_hash->{d};
3608                 $bootindex_hash->{d} += 1;
3609             }
3610         } else {
3611             if ($bootindex_hash->{c}) {
3612                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3613                 $bootindex_hash->{c} += 1;
3614             }
3615         }
3616
3617         if($drive->{interface} eq 'virtio'){
3618            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3619         }
3620
3621         if ($drive->{interface} eq 'scsi') {
3622
3623             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3624
3625             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3626             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3627
3628             my $iothread = '';
3629             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3630                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3631                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3632             } elsif ($drive->{iothread}) {
3633                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3634             }
3635
3636             my $queues = '';
3637             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3638                 $queues = ",num_queues=$drive->{queues}";
3639             }
3640
3641             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3642             $scsicontroller->{$controller}=1;
3643         }
3644
3645         if ($drive->{interface} eq 'sata') {
3646            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3647            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3648            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3649            $ahcicontroller->{$controller}=1;
3650         }
3651
3652         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3653         push @$devices, '-drive',$drive_cmd;
3654         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3655     });
3656
3657     for (my $i = 0; $i < $MAX_NETS; $i++) {
3658          next if !$conf->{"net$i"};
3659          my $d = parse_net($conf->{"net$i"});
3660          next if !$d;
3661
3662          $use_virtio = 1 if $d->{model} eq 'virtio';
3663
3664          if ($bootindex_hash->{n}) {
3665             $d->{bootindex} = $bootindex_hash->{n};
3666             $bootindex_hash->{n} += 1;
3667          }
3668
3669          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3670          push @$devices, '-netdev', $netdevfull;
3671
3672          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3673          push @$devices, '-device', $netdevicefull;
3674     }
3675
3676     if (!$q35) {
3677         # add pci bridges
3678         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3679            $bridges->{1} = 1;
3680            $bridges->{2} = 1;
3681         }
3682
3683         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3684
3685         while (my ($k, $v) = each %$bridges) {
3686             $pciaddr = print_pci_addr("pci.$k");
3687             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3688         }
3689     }
3690
3691     # add custom args
3692     if ($conf->{args}) {
3693         my $aa = PVE::Tools::split_args($conf->{args});
3694         push @$cmd, @$aa;
3695     }
3696
3697     push @$cmd, @$devices;
3698     push @$cmd, '-rtc', join(',', @$rtcFlags)
3699         if scalar(@$rtcFlags);
3700     push @$cmd, '-machine', join(',', @$machineFlags)
3701         if scalar(@$machineFlags);
3702     push @$cmd, '-global', join(',', @$globalFlags)
3703         if scalar(@$globalFlags);
3704
3705     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3706 }
3707
3708 sub vnc_socket {
3709     my ($vmid) = @_;
3710     return "${var_run_tmpdir}/$vmid.vnc";
3711 }
3712
3713 sub spice_port {
3714     my ($vmid) = @_;
3715
3716     my $res = vm_mon_cmd($vmid, 'query-spice');
3717
3718     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3719 }
3720
3721 sub qmp_socket {
3722     my ($vmid, $qga, $name) = @_;
3723     my $sockettype = $qga ? 'qga' : 'qmp';
3724     my $ext = $name ? '-'.$name : '';
3725     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
3726 }
3727
3728 sub pidfile_name {
3729     my ($vmid) = @_;
3730     return "${var_run_tmpdir}/$vmid.pid";
3731 }
3732
3733 sub vm_devices_list {
3734     my ($vmid) = @_;
3735
3736     my $res = vm_mon_cmd($vmid, 'query-pci');
3737     my $devices_to_check = [];
3738     my $devices = {};
3739     foreach my $pcibus (@$res) {
3740         push @$devices_to_check, @{$pcibus->{devices}},
3741     }
3742
3743     while (@$devices_to_check) {
3744         my $to_check = [];
3745         for my $d (@$devices_to_check) {
3746             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3747             next if !$d->{'pci_bridge'};
3748
3749             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3750             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3751         }
3752         $devices_to_check = $to_check;
3753     }
3754
3755     my $resblock = vm_mon_cmd($vmid, 'query-block');
3756     foreach my $block (@$resblock) {
3757         if($block->{device} =~ m/^drive-(\S+)/){
3758                 $devices->{$1} = 1;
3759         }
3760     }
3761
3762     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3763     foreach my $mice (@$resmice) {
3764         if ($mice->{name} eq 'QEMU HID Tablet') {
3765             $devices->{tablet} = 1;
3766             last;
3767         }
3768     }
3769
3770     # for usb devices there is no query-usb
3771     # but we can iterate over the entries in
3772     # qom-list path=/machine/peripheral
3773     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3774     foreach my $per (@$resperipheral) {
3775         if ($per->{name} =~ m/^usb\d+$/) {
3776             $devices->{$per->{name}} = 1;
3777         }
3778     }
3779
3780     return $devices;
3781 }
3782
3783 sub vm_deviceplug {
3784     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3785
3786     my $q35 = machine_type_is_q35($conf);
3787
3788     my $devices_list = vm_devices_list($vmid);
3789     return 1 if defined($devices_list->{$deviceid});
3790
3791     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3792
3793     if ($deviceid eq 'tablet') {
3794
3795         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3796
3797     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3798
3799         die "usb hotplug currently not reliable\n";
3800         # since we can't reliably hot unplug all added usb devices
3801         # and usb passthrough disables live migration
3802         # we disable usb hotplugging for now
3803         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3804
3805     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3806
3807         qemu_iothread_add($vmid, $deviceid, $device);
3808
3809         qemu_driveadd($storecfg, $vmid, $device);
3810         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3811
3812         qemu_deviceadd($vmid, $devicefull);
3813         eval { qemu_deviceaddverify($vmid, $deviceid); };
3814         if (my $err = $@) {
3815             eval { qemu_drivedel($vmid, $deviceid); };
3816             warn $@ if $@;
3817             die $err;
3818         }
3819
3820     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3821
3822
3823         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3824         my $pciaddr = print_pci_addr($deviceid);
3825         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3826
3827         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3828
3829         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3830             qemu_iothread_add($vmid, $deviceid, $device);
3831             $devicefull .= ",iothread=iothread-$deviceid";
3832         }
3833
3834         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3835             $devicefull .= ",num_queues=$device->{queues}";
3836         }
3837
3838         qemu_deviceadd($vmid, $devicefull);
3839         qemu_deviceaddverify($vmid, $deviceid);
3840
3841     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3842
3843         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3844         qemu_driveadd($storecfg, $vmid, $device);
3845
3846         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3847         eval { qemu_deviceadd($vmid, $devicefull); };
3848         if (my $err = $@) {
3849             eval { qemu_drivedel($vmid, $deviceid); };
3850             warn $@ if $@;
3851             die $err;
3852         }
3853
3854     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3855
3856         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3857
3858         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3859         my $use_old_bios_files = undef;
3860         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3861
3862         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3863         qemu_deviceadd($vmid, $netdevicefull);
3864         eval { qemu_deviceaddverify($vmid, $deviceid); };
3865         if (my $err = $@) {
3866             eval { qemu_netdevdel($vmid, $deviceid); };
3867             warn $@ if $@;
3868             die $err;
3869         }
3870
3871     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3872
3873         my $bridgeid = $2;
3874         my $pciaddr = print_pci_addr($deviceid);
3875         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3876
3877         qemu_deviceadd($vmid, $devicefull);
3878         qemu_deviceaddverify($vmid, $deviceid);
3879
3880     } else {
3881         die "can't hotplug device '$deviceid'\n";
3882     }
3883
3884     return 1;
3885 }
3886
3887 # fixme: this should raise exceptions on error!
3888 sub vm_deviceunplug {
3889     my ($vmid, $conf, $deviceid) = @_;
3890
3891     my $devices_list = vm_devices_list($vmid);
3892     return 1 if !defined($devices_list->{$deviceid});
3893
3894     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3895
3896     if ($deviceid eq 'tablet') {
3897
3898         qemu_devicedel($vmid, $deviceid);
3899
3900     } elsif ($deviceid =~ m/^usb\d+$/) {
3901
3902         die "usb hotplug currently not reliable\n";
3903         # when unplugging usb devices this way,
3904         # there may be remaining usb controllers/hubs
3905         # so we disable it for now
3906         qemu_devicedel($vmid, $deviceid);
3907         qemu_devicedelverify($vmid, $deviceid);
3908
3909     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3910
3911         qemu_devicedel($vmid, $deviceid);
3912         qemu_devicedelverify($vmid, $deviceid);
3913         qemu_drivedel($vmid, $deviceid);
3914         qemu_iothread_del($conf, $vmid, $deviceid);
3915
3916     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3917
3918         qemu_devicedel($vmid, $deviceid);
3919         qemu_devicedelverify($vmid, $deviceid);
3920         qemu_iothread_del($conf, $vmid, $deviceid);
3921
3922     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3923
3924         qemu_devicedel($vmid, $deviceid);
3925         qemu_drivedel($vmid, $deviceid);
3926         qemu_deletescsihw($conf, $vmid, $deviceid);
3927
3928     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3929
3930         qemu_devicedel($vmid, $deviceid);
3931         qemu_devicedelverify($vmid, $deviceid);
3932         qemu_netdevdel($vmid, $deviceid);
3933
3934     } else {
3935         die "can't unplug device '$deviceid'\n";
3936     }
3937
3938     return 1;
3939 }
3940
3941 sub qemu_deviceadd {
3942     my ($vmid, $devicefull) = @_;
3943
3944     $devicefull = "driver=".$devicefull;
3945     my %options =  split(/[=,]/, $devicefull);
3946
3947     vm_mon_cmd($vmid, "device_add" , %options);
3948 }
3949
3950 sub qemu_devicedel {
3951     my ($vmid, $deviceid) = @_;
3952
3953     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3954 }
3955
3956 sub qemu_iothread_add {
3957     my($vmid, $deviceid, $device) = @_;
3958
3959     if ($device->{iothread}) {
3960         my $iothreads = vm_iothreads_list($vmid);
3961         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3962     }
3963 }
3964
3965 sub qemu_iothread_del {
3966     my($conf, $vmid, $deviceid) = @_;
3967
3968     my $device = parse_drive($deviceid, $conf->{$deviceid});
3969     if ($device->{iothread}) {
3970         my $iothreads = vm_iothreads_list($vmid);
3971         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3972     }
3973 }
3974
3975 sub qemu_objectadd {
3976     my($vmid, $objectid, $qomtype) = @_;
3977
3978     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3979
3980     return 1;
3981 }
3982
3983 sub qemu_objectdel {
3984     my($vmid, $objectid) = @_;
3985
3986     vm_mon_cmd($vmid, "object-del", id => $objectid);
3987
3988     return 1;
3989 }
3990
3991 sub qemu_driveadd {
3992     my ($storecfg, $vmid, $device) = @_;
3993
3994     my $drive = print_drive_full($storecfg, $vmid, $device);
3995     $drive =~ s/\\/\\\\/g;
3996     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3997
3998     # If the command succeeds qemu prints: "OK"
3999     return 1 if $ret =~ m/OK/s;
4000
4001     die "adding drive failed: $ret\n";
4002 }
4003
4004 sub qemu_drivedel {
4005     my($vmid, $deviceid) = @_;
4006
4007     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4008     $ret =~ s/^\s+//;
4009
4010     return 1 if $ret eq "";
4011
4012     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4013     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4014
4015     die "deleting drive $deviceid failed : $ret\n";
4016 }
4017
4018 sub qemu_deviceaddverify {
4019     my ($vmid, $deviceid) = @_;
4020
4021     for (my $i = 0; $i <= 5; $i++) {
4022          my $devices_list = vm_devices_list($vmid);
4023          return 1 if defined($devices_list->{$deviceid});
4024          sleep 1;
4025     }
4026
4027     die "error on hotplug device '$deviceid'\n";
4028 }
4029
4030
4031 sub qemu_devicedelverify {
4032     my ($vmid, $deviceid) = @_;
4033
4034     # need to verify that the device is correctly removed as device_del
4035     # is async and empty return is not reliable
4036
4037     for (my $i = 0; $i <= 5; $i++) {
4038          my $devices_list = vm_devices_list($vmid);
4039          return 1 if !defined($devices_list->{$deviceid});
4040          sleep 1;
4041     }
4042
4043     die "error on hot-unplugging device '$deviceid'\n";
4044 }
4045
4046 sub qemu_findorcreatescsihw {
4047     my ($storecfg, $conf, $vmid, $device) = @_;
4048
4049     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4050
4051     my $scsihwid="$controller_prefix$controller";
4052     my $devices_list = vm_devices_list($vmid);
4053
4054     if(!defined($devices_list->{$scsihwid})) {
4055         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4056     }
4057
4058     return 1;
4059 }
4060
4061 sub qemu_deletescsihw {
4062     my ($conf, $vmid, $opt) = @_;
4063
4064     my $device = parse_drive($opt, $conf->{$opt});
4065
4066     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4067         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4068         return 1;
4069     }
4070
4071     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4072
4073     my $devices_list = vm_devices_list($vmid);
4074     foreach my $opt (keys %{$devices_list}) {
4075         if (PVE::QemuServer::is_valid_drivename($opt)) {
4076             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4077             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4078                 return 1;
4079             }
4080         }
4081     }
4082
4083     my $scsihwid="scsihw$controller";
4084
4085     vm_deviceunplug($vmid, $conf, $scsihwid);
4086
4087     return 1;
4088 }
4089
4090 sub qemu_add_pci_bridge {
4091     my ($storecfg, $conf, $vmid, $device) = @_;
4092
4093     my $bridges = {};
4094
4095     my $bridgeid;
4096
4097     print_pci_addr($device, $bridges);
4098
4099     while (my ($k, $v) = each %$bridges) {
4100         $bridgeid = $k;
4101     }
4102     return 1 if !defined($bridgeid) || $bridgeid < 1;
4103
4104     my $bridge = "pci.$bridgeid";
4105     my $devices_list = vm_devices_list($vmid);
4106
4107     if (!defined($devices_list->{$bridge})) {
4108         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4109     }
4110
4111     return 1;
4112 }
4113
4114 sub qemu_set_link_status {
4115     my ($vmid, $device, $up) = @_;
4116
4117     vm_mon_cmd($vmid, "set_link", name => $device,
4118                up => $up ? JSON::true : JSON::false);
4119 }
4120
4121 sub qemu_netdevadd {
4122     my ($vmid, $conf, $device, $deviceid) = @_;
4123
4124     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4125     my %options =  split(/[=,]/, $netdev);
4126
4127     vm_mon_cmd($vmid, "netdev_add",  %options);
4128     return 1;
4129 }
4130
4131 sub qemu_netdevdel {
4132     my ($vmid, $deviceid) = @_;
4133
4134     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4135 }
4136
4137 sub qemu_usb_hotplug {
4138     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4139
4140     return if !$device;
4141
4142     # remove the old one first
4143     vm_deviceunplug($vmid, $conf, $deviceid);
4144
4145     # check if xhci controller is necessary and available
4146     if ($device->{usb3}) {
4147
4148         my $devicelist = vm_devices_list($vmid);
4149
4150         if (!$devicelist->{xhci}) {
4151             my $pciaddr = print_pci_addr("xhci");
4152             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4153         }
4154     }
4155     my $d = parse_usb_device($device->{host});
4156     $d->{usb3} = $device->{usb3};
4157
4158     # add the new one
4159     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4160 }
4161
4162 sub qemu_cpu_hotplug {
4163     my ($vmid, $conf, $vcpus) = @_;
4164
4165     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4166
4167     my $sockets = 1;
4168     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4169     $sockets = $conf->{sockets} if  $conf->{sockets};
4170     my $cores = $conf->{cores} || 1;
4171     my $maxcpus = $sockets * $cores;
4172
4173     $vcpus = $maxcpus if !$vcpus;
4174
4175     die "you can't add more vcpus than maxcpus\n"
4176         if $vcpus > $maxcpus;
4177
4178     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4179
4180     if ($vcpus < $currentvcpus) {
4181
4182         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4183
4184             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4185                 qemu_devicedel($vmid, "cpu$i");
4186                 my $retry = 0;
4187                 my $currentrunningvcpus = undef;
4188                 while (1) {
4189                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4190                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4191                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4192                     $retry++;
4193                     sleep 1;
4194                 }
4195                 #update conf after each succesfull cpu unplug
4196                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4197                 PVE::QemuConfig->write_config($vmid, $conf);
4198             }
4199         } else {
4200             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4201         }
4202
4203         return;
4204     }
4205
4206     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4207     die "vcpus in running vm does not match its configuration\n"
4208         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4209
4210     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4211
4212         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4213             my $cpustr = print_cpu_device($conf, $i);
4214             qemu_deviceadd($vmid, $cpustr);
4215
4216             my $retry = 0;
4217             my $currentrunningvcpus = undef;
4218             while (1) {
4219                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4220                 last if scalar(@{$currentrunningvcpus}) == $i;
4221                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4222                 sleep 1;
4223                 $retry++;
4224             }
4225             #update conf after each succesfull cpu hotplug
4226             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4227             PVE::QemuConfig->write_config($vmid, $conf);
4228         }
4229     } else {
4230
4231         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4232             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4233         }
4234     }
4235 }
4236
4237 sub qemu_block_set_io_throttle {
4238     my ($vmid, $deviceid,
4239         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4240         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4241         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4242         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4243
4244     return if !check_running($vmid) ;
4245
4246     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4247         bps => int($bps),
4248         bps_rd => int($bps_rd),
4249         bps_wr => int($bps_wr),
4250         iops => int($iops),
4251         iops_rd => int($iops_rd),
4252         iops_wr => int($iops_wr),
4253         bps_max => int($bps_max),
4254         bps_rd_max => int($bps_rd_max),
4255         bps_wr_max => int($bps_wr_max),
4256         iops_max => int($iops_max),
4257         iops_rd_max => int($iops_rd_max),
4258         iops_wr_max => int($iops_wr_max),
4259         bps_max_length => int($bps_max_length),
4260         bps_rd_max_length => int($bps_rd_max_length),
4261         bps_wr_max_length => int($bps_wr_max_length),
4262         iops_max_length => int($iops_max_length),
4263         iops_rd_max_length => int($iops_rd_max_length),
4264         iops_wr_max_length => int($iops_wr_max_length),
4265     );
4266
4267 }
4268
4269 # old code, only used to shutdown old VM after update
4270 sub __read_avail {
4271     my ($fh, $timeout) = @_;
4272
4273     my $sel = new IO::Select;
4274     $sel->add($fh);
4275
4276     my $res = '';
4277     my $buf;
4278
4279     my @ready;
4280     while (scalar (@ready = $sel->can_read($timeout))) {
4281         my $count;
4282         if ($count = $fh->sysread($buf, 8192)) {
4283             if ($buf =~ /^(.*)\(qemu\) $/s) {
4284                 $res .= $1;
4285                 last;
4286             } else {
4287                 $res .= $buf;
4288             }
4289         } else {
4290             if (!defined($count)) {
4291                 die "$!\n";
4292             }
4293             last;
4294         }
4295     }
4296
4297     die "monitor read timeout\n" if !scalar(@ready);
4298
4299     return $res;
4300 }
4301
4302 sub qemu_block_resize {
4303     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4304
4305     my $running = check_running($vmid);
4306
4307     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4308
4309     return if !$running;
4310
4311     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4312
4313 }
4314
4315 sub qemu_volume_snapshot {
4316     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4317
4318     my $running = check_running($vmid);
4319
4320     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4321         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4322     } else {
4323         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4324     }
4325 }
4326
4327 sub qemu_volume_snapshot_delete {
4328     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4329
4330     my $running = check_running($vmid);
4331
4332     if($running) {
4333
4334         $running = undef;
4335         my $conf = PVE::QemuConfig->load_config($vmid);
4336         foreach_drive($conf, sub {
4337             my ($ds, $drive) = @_;
4338             $running = 1 if $drive->{file} eq $volid;
4339         });
4340     }
4341
4342     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4343         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4344     } else {
4345         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4346     }
4347 }
4348
4349 sub set_migration_caps {
4350     my ($vmid) = @_;
4351
4352     my $cap_ref = [];
4353
4354     my $enabled_cap = {
4355         "auto-converge" => 1,
4356         "xbzrle" => 1,
4357         "x-rdma-pin-all" => 0,
4358         "zero-blocks" => 0,
4359         "compress" => 0
4360     };
4361
4362     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4363
4364     for my $supported_capability (@$supported_capabilities) {
4365         push @$cap_ref, {
4366             capability => $supported_capability->{capability},
4367             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4368         };
4369     }
4370
4371     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4372 }
4373
4374 my $fast_plug_option = {
4375     'lock' => 1,
4376     'name' => 1,
4377     'onboot' => 1,
4378     'shares' => 1,
4379     'startup' => 1,
4380     'description' => 1,
4381     'protection' => 1,
4382     'vmstatestorage' => 1,
4383 };
4384
4385 # hotplug changes in [PENDING]
4386 # $selection hash can be used to only apply specified options, for
4387 # example: { cores => 1 } (only apply changed 'cores')
4388 # $errors ref is used to return error messages
4389 sub vmconfig_hotplug_pending {
4390     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4391
4392     my $defaults = load_defaults();
4393
4394     # commit values which do not have any impact on running VM first
4395     # Note: those option cannot raise errors, we we do not care about
4396     # $selection and always apply them.
4397
4398     my $add_error = sub {
4399         my ($opt, $msg) = @_;
4400         $errors->{$opt} = "hotplug problem - $msg";
4401     };
4402
4403     my $changes = 0;
4404     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4405         if ($fast_plug_option->{$opt}) {
4406             $conf->{$opt} = $conf->{pending}->{$opt};
4407             delete $conf->{pending}->{$opt};
4408             $changes = 1;
4409         }
4410     }
4411
4412     if ($changes) {
4413         PVE::QemuConfig->write_config($vmid, $conf);
4414         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4415     }
4416
4417     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4418
4419     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4420     while (my ($opt, $force) = each %$pending_delete_hash) {
4421         next if $selection && !$selection->{$opt};
4422         eval {
4423             if ($opt eq 'hotplug') {
4424                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4425             } elsif ($opt eq 'tablet') {
4426                 die "skip\n" if !$hotplug_features->{usb};
4427                 if ($defaults->{tablet}) {
4428                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4429                 } else {
4430                     vm_deviceunplug($vmid, $conf, $opt);
4431                 }
4432             } elsif ($opt =~ m/^usb\d+/) {
4433                 die "skip\n";
4434                 # since we cannot reliably hot unplug usb devices
4435                 # we are disabling it
4436                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4437                 vm_deviceunplug($vmid, $conf, $opt);
4438             } elsif ($opt eq 'vcpus') {
4439                 die "skip\n" if !$hotplug_features->{cpu};
4440                 qemu_cpu_hotplug($vmid, $conf, undef);
4441             } elsif ($opt eq 'balloon') {
4442                 # enable balloon device is not hotpluggable
4443                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4444                 # here we reset the ballooning value to memory
4445                 my $balloon = $conf->{memory} || $defaults->{memory};
4446                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4447             } elsif ($fast_plug_option->{$opt}) {
4448                 # do nothing
4449             } elsif ($opt =~ m/^net(\d+)$/) {
4450                 die "skip\n" if !$hotplug_features->{network};
4451                 vm_deviceunplug($vmid, $conf, $opt);
4452             } elsif (is_valid_drivename($opt)) {
4453                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4454                 vm_deviceunplug($vmid, $conf, $opt);
4455                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4456             } elsif ($opt =~ m/^memory$/) {
4457                 die "skip\n" if !$hotplug_features->{memory};
4458                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4459             } elsif ($opt eq 'cpuunits') {
4460                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4461             } elsif ($opt eq 'cpulimit') {
4462                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4463             } else {
4464                 die "skip\n";
4465             }
4466         };
4467         if (my $err = $@) {
4468             &$add_error($opt, $err) if $err ne "skip\n";
4469         } else {
4470             # save new config if hotplug was successful
4471             delete $conf->{$opt};
4472             vmconfig_undelete_pending_option($conf, $opt);
4473             PVE::QemuConfig->write_config($vmid, $conf);
4474             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4475         }
4476     }
4477
4478     my $apply_pending_cloudinit;
4479     $apply_pending_cloudinit = sub {
4480         my ($key, $value) = @_;
4481         $apply_pending_cloudinit = sub {}; # once is enough
4482
4483         my @cloudinit_opts = keys %$confdesc_cloudinit;
4484         foreach my $opt (keys %{$conf->{pending}}) {
4485             next if !grep { $_ eq $opt } @cloudinit_opts;
4486             $conf->{$opt} = delete $conf->{pending}->{$opt};
4487         }
4488
4489         my $new_conf = { %$conf };
4490         $new_conf->{$key} = $value;
4491         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4492     };
4493
4494     foreach my $opt (keys %{$conf->{pending}}) {
4495         next if $selection && !$selection->{$opt};
4496         my $value = $conf->{pending}->{$opt};
4497         eval {
4498             if ($opt eq 'hotplug') {
4499                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4500             } elsif ($opt eq 'tablet') {
4501                 die "skip\n" if !$hotplug_features->{usb};
4502                 if ($value == 1) {
4503                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4504                 } elsif ($value == 0) {
4505                     vm_deviceunplug($vmid, $conf, $opt);
4506                 }
4507             } elsif ($opt =~ m/^usb\d+$/) {
4508                 die "skip\n";
4509                 # since we cannot reliably hot unplug usb devices
4510                 # we are disabling it
4511                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4512                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4513                 die "skip\n" if !$d;
4514                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4515             } elsif ($opt eq 'vcpus') {
4516                 die "skip\n" if !$hotplug_features->{cpu};
4517                 qemu_cpu_hotplug($vmid, $conf, $value);
4518             } elsif ($opt eq 'balloon') {
4519                 # enable/disable balloning device is not hotpluggable
4520                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4521                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4522                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4523
4524                 # allow manual ballooning if shares is set to zero
4525                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4526                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4527                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4528                 }
4529             } elsif ($opt =~ m/^net(\d+)$/) {
4530                 # some changes can be done without hotplug
4531                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4532                                     $vmid, $opt, $value);
4533             } elsif (is_valid_drivename($opt)) {
4534                 # some changes can be done without hotplug
4535                 my $drive = parse_drive($opt, $value);
4536                 if (drive_is_cloudinit($drive)) {
4537                     &$apply_pending_cloudinit($opt, $value);
4538                 }
4539                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4540                                      $vmid, $opt, $value, 1);
4541             } elsif ($opt =~ m/^memory$/) { #dimms
4542                 die "skip\n" if !$hotplug_features->{memory};
4543                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4544             } elsif ($opt eq 'cpuunits') {
4545                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4546             } elsif ($opt eq 'cpulimit') {
4547                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4548                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4549             } else {
4550                 die "skip\n";  # skip non-hot-pluggable options
4551             }
4552         };
4553         if (my $err = $@) {
4554             &$add_error($opt, $err) if $err ne "skip\n";
4555         } else {
4556             # save new config if hotplug was successful
4557             $conf->{$opt} = $value;
4558             delete $conf->{pending}->{$opt};
4559             PVE::QemuConfig->write_config($vmid, $conf);
4560             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4561         }
4562     }
4563 }
4564
4565 sub try_deallocate_drive {
4566     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4567
4568     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4569         my $volid = $drive->{file};
4570         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4571             my $sid = PVE::Storage::parse_volume_id($volid);
4572             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4573
4574             # check if the disk is really unused
4575             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4576                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4577             PVE::Storage::vdisk_free($storecfg, $volid);
4578             return 1;
4579         } else {
4580             # If vm is not owner of this disk remove from config
4581             return 1;
4582         }
4583     }
4584
4585     return undef;
4586 }
4587
4588 sub vmconfig_delete_or_detach_drive {
4589     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4590
4591     my $drive = parse_drive($opt, $conf->{$opt});
4592
4593     my $rpcenv = PVE::RPCEnvironment::get();
4594     my $authuser = $rpcenv->get_user();
4595
4596     if ($force) {
4597         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4598         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4599     } else {
4600         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4601     }
4602 }
4603
4604 sub vmconfig_apply_pending {
4605     my ($vmid, $conf, $storecfg) = @_;
4606
4607     # cold plug
4608
4609     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4610     while (my ($opt, $force) = each %$pending_delete_hash) {
4611         die "internal error" if $opt =~ m/^unused/;
4612         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4613         if (!defined($conf->{$opt})) {
4614             vmconfig_undelete_pending_option($conf, $opt);
4615             PVE::QemuConfig->write_config($vmid, $conf);
4616         } elsif (is_valid_drivename($opt)) {
4617             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4618             vmconfig_undelete_pending_option($conf, $opt);
4619             delete $conf->{$opt};
4620             PVE::QemuConfig->write_config($vmid, $conf);
4621         } else {
4622             vmconfig_undelete_pending_option($conf, $opt);
4623             delete $conf->{$opt};
4624             PVE::QemuConfig->write_config($vmid, $conf);
4625         }
4626     }
4627
4628     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4629
4630     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4631         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4632
4633         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4634             # skip if nothing changed
4635         } elsif (is_valid_drivename($opt)) {
4636             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4637                 if defined($conf->{$opt});
4638             $conf->{$opt} = $conf->{pending}->{$opt};
4639         } else {
4640             $conf->{$opt} = $conf->{pending}->{$opt};
4641         }
4642
4643         delete $conf->{pending}->{$opt};
4644         PVE::QemuConfig->write_config($vmid, $conf);
4645     }
4646 }
4647
4648 my $safe_num_ne = sub {
4649     my ($a, $b) = @_;
4650
4651     return 0 if !defined($a) && !defined($b);
4652     return 1 if !defined($a);
4653     return 1 if !defined($b);
4654
4655     return $a != $b;
4656 };
4657
4658 my $safe_string_ne = sub {
4659     my ($a, $b) = @_;
4660
4661     return 0 if !defined($a) && !defined($b);
4662     return 1 if !defined($a);
4663     return 1 if !defined($b);
4664
4665     return $a ne $b;
4666 };
4667
4668 sub vmconfig_update_net {
4669     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4670
4671     my $newnet = parse_net($value);
4672
4673     if ($conf->{$opt}) {
4674         my $oldnet = parse_net($conf->{$opt});
4675
4676         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4677             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4678             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4679             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4680
4681             # for non online change, we try to hot-unplug
4682             die "skip\n" if !$hotplug;
4683             vm_deviceunplug($vmid, $conf, $opt);
4684         } else {
4685
4686             die "internal error" if $opt !~ m/net(\d+)/;
4687             my $iface = "tap${vmid}i$1";
4688
4689             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4690                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4691                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4692                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4693                 PVE::Network::tap_unplug($iface);
4694                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4695             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4696                 # Rate can be applied on its own but any change above needs to
4697                 # include the rate in tap_plug since OVS resets everything.
4698                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4699             }
4700
4701             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4702                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4703             }
4704
4705             return 1;
4706         }
4707     }
4708
4709     if ($hotplug) {
4710         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4711     } else {
4712         die "skip\n";
4713     }
4714 }
4715
4716 sub vmconfig_update_disk {
4717     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4718
4719     # fixme: do we need force?
4720
4721     my $drive = parse_drive($opt, $value);
4722
4723     if ($conf->{$opt}) {
4724
4725         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4726
4727             my $media = $drive->{media} || 'disk';
4728             my $oldmedia = $old_drive->{media} || 'disk';
4729             die "unable to change media type\n" if $media ne $oldmedia;
4730
4731             if (!drive_is_cdrom($old_drive)) {
4732
4733                 if ($drive->{file} ne $old_drive->{file}) {
4734
4735                     die "skip\n" if !$hotplug;
4736
4737                     # unplug and register as unused
4738                     vm_deviceunplug($vmid, $conf, $opt);
4739                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4740
4741                 } else {
4742                     # update existing disk
4743
4744                     # skip non hotpluggable value
4745                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4746                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4747                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4748                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4749                         die "skip\n";
4750                     }
4751
4752                     # apply throttle
4753                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4754                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4755                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4756                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4757                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4758                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4759                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4760                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4761                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4762                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4763                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4764                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4765                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4766                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4767                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4768                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4769                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4770                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4771
4772                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4773                                                    ($drive->{mbps} || 0)*1024*1024,
4774                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4775                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4776                                                    $drive->{iops} || 0,
4777                                                    $drive->{iops_rd} || 0,
4778                                                    $drive->{iops_wr} || 0,
4779                                                    ($drive->{mbps_max} || 0)*1024*1024,
4780                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4781                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4782                                                    $drive->{iops_max} || 0,
4783                                                    $drive->{iops_rd_max} || 0,
4784                                                    $drive->{iops_wr_max} || 0,
4785                                                    $drive->{bps_max_length} || 1,
4786                                                    $drive->{bps_rd_max_length} || 1,
4787                                                    $drive->{bps_wr_max_length} || 1,
4788                                                    $drive->{iops_max_length} || 1,
4789                                                    $drive->{iops_rd_max_length} || 1,
4790                                                    $drive->{iops_wr_max_length} || 1);
4791
4792                     }
4793
4794                     return 1;
4795                 }
4796
4797             } else { # cdrom
4798
4799                 if ($drive->{file} eq 'none') {
4800                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4801                     if (drive_is_cloudinit($old_drive)) {
4802                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4803                     }
4804                 } else {
4805                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4806                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4807                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4808                 }
4809
4810                 return 1;
4811             }
4812         }
4813     }
4814
4815     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4816     # hotplug new disks
4817     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4818     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4819 }
4820
4821 sub vm_start {
4822     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4823         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4824
4825     PVE::QemuConfig->lock_config($vmid, sub {
4826         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4827
4828         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4829
4830         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4831
4832         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4833
4834         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4835             vmconfig_apply_pending($vmid, $conf, $storecfg);
4836             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4837         }
4838
4839         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4840
4841         my $defaults = load_defaults();
4842
4843         # set environment variable useful inside network script
4844         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4845
4846         my $local_volumes = {};
4847
4848         if ($targetstorage) {
4849             foreach_drive($conf, sub {
4850                 my ($ds, $drive) = @_;
4851
4852                 return if drive_is_cdrom($drive);
4853
4854                 my $volid = $drive->{file};
4855
4856                 return if !$volid;
4857
4858                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4859
4860                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4861                 return if $scfg->{shared};
4862                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4863             });
4864
4865             my $format = undef;
4866
4867             foreach my $opt (sort keys %$local_volumes) {
4868
4869                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4870                 my $drive = parse_drive($opt, $conf->{$opt});
4871
4872                 #if remote storage is specified, use default format
4873                 if ($targetstorage && $targetstorage ne "1") {
4874                     $storeid = $targetstorage;
4875                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4876                     $format = $defFormat;
4877                 } else {
4878                     #else we use same format than original
4879                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4880                     $format = qemu_img_format($scfg, $volid);
4881                 }
4882
4883                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4884                 my $newdrive = $drive;
4885                 $newdrive->{format} = $format;
4886                 $newdrive->{file} = $newvolid;
4887                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4888                 $local_volumes->{$opt} = $drivestr;
4889                 #pass drive to conf for command line
4890                 $conf->{$opt} = $drivestr;
4891             }
4892         }
4893
4894         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4895
4896         my $migrate_port = 0;
4897         my $migrate_uri;
4898         if ($statefile) {
4899             if ($statefile eq 'tcp') {
4900                 my $localip = "localhost";
4901                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4902                 my $nodename = PVE::INotify::nodename();
4903
4904                 if (!defined($migration_type)) {
4905                     if (defined($datacenterconf->{migration}->{type})) {
4906                         $migration_type = $datacenterconf->{migration}->{type};
4907                     } else {
4908                         $migration_type = 'secure';
4909                     }
4910                 }
4911
4912                 if ($migration_type eq 'insecure') {
4913                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4914                     if ($migrate_network_addr) {
4915                         $localip = $migrate_network_addr;
4916                     } else {
4917                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4918                     }
4919
4920                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4921                 }
4922
4923                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4924                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4925                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4926                 push @$cmd, '-incoming', $migrate_uri;
4927                 push @$cmd, '-S';
4928
4929             } elsif ($statefile eq 'unix') {
4930                 # should be default for secure migrations as a ssh TCP forward
4931                 # tunnel is not deterministic reliable ready and fails regurarly
4932                 # to set up in time, so use UNIX socket forwards
4933                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4934                 unlink $socket_addr;
4935
4936                 $migrate_uri = "unix:$socket_addr";
4937
4938                 push @$cmd, '-incoming', $migrate_uri;
4939                 push @$cmd, '-S';
4940
4941             } else {
4942                 push @$cmd, '-loadstate', $statefile;
4943             }
4944         } elsif ($paused) {
4945             push @$cmd, '-S';
4946         }
4947
4948         # host pci devices
4949         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4950           my $d = parse_hostpci($conf->{"hostpci$i"});
4951           next if !$d;
4952           my $pcidevices = $d->{pciid};
4953           foreach my $pcidevice (@$pcidevices) {
4954                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4955
4956                 my $info = pci_device_info("0000:$pciid");
4957                 die "IOMMU not present\n" if !check_iommu_support();
4958                 die "no pci device info for device '$pciid'\n" if !$info;
4959                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4960                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4961           }
4962         }
4963
4964         PVE::Storage::activate_volumes($storecfg, $vollist);
4965
4966         if (!check_running($vmid, 1)) {
4967             eval {
4968                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4969                     outfunc => sub {}, errfunc => sub {});
4970             };
4971         }
4972
4973         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4974                                                   : $defaults->{cpuunits};
4975
4976         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4977         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4978
4979         my %properties = (
4980             Slice => 'qemu.slice',
4981             KillMode => 'none',
4982             CPUShares => $cpuunits
4983         );
4984
4985         if (my $cpulimit = $conf->{cpulimit}) {
4986             $properties{CPUQuota} = int($cpulimit * 100);
4987         }
4988         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4989
4990         my $run_qemu = sub {
4991             PVE::Tools::run_fork sub {
4992                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4993                 run_command($cmd, %run_params);
4994             };
4995         };
4996
4997         if ($conf->{hugepages}) {
4998
4999             my $code = sub {
5000                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5001                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5002
5003                 PVE::QemuServer::Memory::hugepages_mount();
5004                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5005
5006                 eval { $run_qemu->() };
5007                 if (my $err = $@) {
5008                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5009                     die $err;
5010                 }
5011
5012                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5013             };
5014             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5015
5016         } else {
5017             eval { $run_qemu->() };
5018         }
5019
5020         if (my $err = $@) {
5021             # deactivate volumes if start fails
5022             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5023             die "start failed: $err";
5024         }
5025
5026         print "migration listens on $migrate_uri\n" if $migrate_uri;
5027
5028         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5029             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5030             warn $@ if $@;
5031         }
5032
5033         #start nbd server for storage migration
5034         if ($targetstorage) {
5035             my $nodename = PVE::INotify::nodename();
5036             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5037             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5038             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5039             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5040
5041             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5042
5043             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5044
5045             foreach my $opt (sort keys %$local_volumes) {
5046                 my $volid = $local_volumes->{$opt};
5047                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5048                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5049                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5050             }
5051         }
5052
5053         if ($migratedfrom) {
5054             eval {
5055                 set_migration_caps($vmid);
5056             };
5057             warn $@ if $@;
5058
5059             if ($spice_port) {
5060                 print "spice listens on port $spice_port\n";
5061                 if ($spice_ticket) {
5062                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5063                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5064                 }
5065             }
5066
5067         } else {
5068             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5069                 if !$statefile && $conf->{balloon};
5070
5071             foreach my $opt (keys %$conf) {
5072                 next if $opt !~  m/^net\d+$/;
5073                 my $nicconf = parse_net($conf->{$opt});
5074                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5075             }
5076         }
5077
5078         vm_mon_cmd_nocheck($vmid, 'qom-set',
5079                     path => "machine/peripheral/balloon0",
5080                     property => "guest-stats-polling-interval",
5081                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5082
5083     });
5084 }
5085
5086 sub vm_mon_cmd {
5087     my ($vmid, $execute, %params) = @_;
5088
5089     my $cmd = { execute => $execute, arguments => \%params };
5090     vm_qmp_command($vmid, $cmd);
5091 }
5092
5093 sub vm_mon_cmd_nocheck {
5094     my ($vmid, $execute, %params) = @_;
5095
5096     my $cmd = { execute => $execute, arguments => \%params };
5097     vm_qmp_command($vmid, $cmd, 1);
5098 }
5099
5100 sub vm_qmp_command {
5101     my ($vmid, $cmd, $nocheck) = @_;
5102
5103     my $res;
5104
5105     my $timeout;
5106     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5107         $timeout = $cmd->{arguments}->{timeout};
5108         delete $cmd->{arguments}->{timeout};
5109     }
5110
5111     eval {
5112         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5113         my $sname = qmp_socket($vmid);
5114         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5115             my $qmpclient = PVE::QMPClient->new();
5116
5117             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5118         } else {
5119             die "unable to open monitor socket\n";
5120         }
5121     };
5122     if (my $err = $@) {
5123         syslog("err", "VM $vmid qmp command failed - $err");
5124         die $err;
5125     }
5126
5127     return $res;
5128 }
5129
5130 sub vm_human_monitor_command {
5131     my ($vmid, $cmdline) = @_;
5132
5133     my $res;
5134
5135     my $cmd = {
5136         execute => 'human-monitor-command',
5137         arguments => { 'command-line' => $cmdline},
5138     };
5139
5140     return vm_qmp_command($vmid, $cmd);
5141 }
5142
5143 sub vm_commandline {
5144     my ($storecfg, $vmid) = @_;
5145
5146     my $conf = PVE::QemuConfig->load_config($vmid);
5147
5148     my $defaults = load_defaults();
5149
5150     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5151
5152     return PVE::Tools::cmd2string($cmd);
5153 }
5154
5155 sub vm_reset {
5156     my ($vmid, $skiplock) = @_;
5157
5158     PVE::QemuConfig->lock_config($vmid, sub {
5159
5160         my $conf = PVE::QemuConfig->load_config($vmid);
5161
5162         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5163
5164         vm_mon_cmd($vmid, "system_reset");
5165     });
5166 }
5167
5168 sub get_vm_volumes {
5169     my ($conf) = @_;
5170
5171     my $vollist = [];
5172     foreach_volid($conf, sub {
5173         my ($volid, $attr) = @_;
5174
5175         return if $volid =~ m|^/|;
5176
5177         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5178         return if !$sid;
5179
5180         push @$vollist, $volid;
5181     });
5182
5183     return $vollist;
5184 }
5185
5186 sub vm_stop_cleanup {
5187     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5188
5189     eval {
5190
5191         if (!$keepActive) {
5192             my $vollist = get_vm_volumes($conf);
5193             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5194         }
5195
5196         foreach my $ext (qw(mon qmp pid vnc qga)) {
5197             unlink "/var/run/qemu-server/${vmid}.$ext";
5198         }
5199
5200         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5201     };
5202     warn $@ if $@; # avoid errors - just warn
5203 }
5204
5205 # Note: use $nockeck to skip tests if VM configuration file exists.
5206 # We need that when migration VMs to other nodes (files already moved)
5207 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5208 sub vm_stop {
5209     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5210
5211     $force = 1 if !defined($force) && !$shutdown;
5212
5213     if ($migratedfrom){
5214         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5215         kill 15, $pid if $pid;
5216         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5217         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5218         return;
5219     }
5220
5221     PVE::QemuConfig->lock_config($vmid, sub {
5222
5223         my $pid = check_running($vmid, $nocheck);
5224         return if !$pid;
5225
5226         my $conf;
5227         if (!$nocheck) {
5228             $conf = PVE::QemuConfig->load_config($vmid);
5229             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5230             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5231                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5232                 $timeout = $opts->{down} if $opts->{down};
5233             }
5234         }
5235
5236         $timeout = 60 if !defined($timeout);
5237
5238         eval {
5239             if ($shutdown) {
5240                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5241                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5242                 } else {
5243                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5244                 }
5245             } else {
5246                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5247             }
5248         };
5249         my $err = $@;
5250
5251         if (!$err) {
5252             my $count = 0;
5253             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5254                 $count++;
5255                 sleep 1;
5256             }
5257
5258             if ($count >= $timeout) {
5259                 if ($force) {
5260                     warn "VM still running - terminating now with SIGTERM\n";
5261                     kill 15, $pid;
5262                 } else {
5263                     die "VM quit/powerdown failed - got timeout\n";
5264                 }
5265             } else {
5266                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5267                 return;
5268             }
5269         } else {
5270             if ($force) {
5271                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5272                 kill 15, $pid;
5273             } else {
5274                 die "VM quit/powerdown failed\n";
5275             }
5276         }
5277
5278         # wait again
5279         $timeout = 10;
5280
5281         my $count = 0;
5282         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5283             $count++;
5284             sleep 1;
5285         }
5286
5287         if ($count >= $timeout) {
5288             warn "VM still running - terminating now with SIGKILL\n";
5289             kill 9, $pid;
5290             sleep 1;
5291         }
5292
5293         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5294    });
5295 }
5296
5297 sub vm_suspend {
5298     my ($vmid, $skiplock) = @_;
5299
5300     PVE::QemuConfig->lock_config($vmid, sub {
5301
5302         my $conf = PVE::QemuConfig->load_config($vmid);
5303
5304         PVE::QemuConfig->check_lock($conf)
5305             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5306
5307         vm_mon_cmd($vmid, "stop");
5308     });
5309 }
5310
5311 sub vm_resume {
5312     my ($vmid, $skiplock, $nocheck) = @_;
5313
5314     PVE::QemuConfig->lock_config($vmid, sub {
5315
5316         my $res = vm_mon_cmd($vmid, 'query-status');
5317         my $resume_cmd = 'cont';
5318
5319         if ($res->{status} && $res->{status} eq 'suspended') {
5320             $resume_cmd = 'system_wakeup';
5321         }
5322
5323         if (!$nocheck) {
5324
5325             my $conf = PVE::QemuConfig->load_config($vmid);
5326
5327             PVE::QemuConfig->check_lock($conf)
5328                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5329
5330             vm_mon_cmd($vmid, $resume_cmd);
5331
5332         } else {
5333             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5334         }
5335     });
5336 }
5337
5338 sub vm_sendkey {
5339     my ($vmid, $skiplock, $key) = @_;
5340
5341     PVE::QemuConfig->lock_config($vmid, sub {
5342
5343         my $conf = PVE::QemuConfig->load_config($vmid);
5344
5345         # there is no qmp command, so we use the human monitor command
5346         vm_human_monitor_command($vmid, "sendkey $key");
5347     });
5348 }
5349
5350 sub vm_destroy {
5351     my ($storecfg, $vmid, $skiplock) = @_;
5352
5353     PVE::QemuConfig->lock_config($vmid, sub {
5354
5355         my $conf = PVE::QemuConfig->load_config($vmid);
5356
5357         if (!check_running($vmid)) {
5358             destroy_vm($storecfg, $vmid, undef, $skiplock);
5359         } else {
5360             die "VM $vmid is running - destroy failed\n";
5361         }
5362     });
5363 }
5364
5365 # pci helpers
5366
5367 sub file_write {
5368     my ($filename, $buf) = @_;
5369
5370     my $fh = IO::File->new($filename, "w");
5371     return undef if !$fh;
5372
5373     my $res = print $fh $buf;
5374
5375     $fh->close();
5376
5377     return $res;
5378 }
5379
5380 sub pci_device_info {
5381     my ($name) = @_;
5382
5383     my $res;
5384
5385     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5386     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5387
5388     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5389     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5390
5391     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5392     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5393
5394     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5395     return undef if !defined($product) || $product !~ s/^0x//;
5396
5397     $res = {
5398         name => $name,
5399         vendor => $vendor,
5400         product => $product,
5401         domain => $domain,
5402         bus => $bus,
5403         slot => $slot,
5404         func => $func,
5405         irq => $irq,
5406         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5407     };
5408
5409     return $res;
5410 }
5411
5412 sub pci_dev_reset {
5413     my ($dev) = @_;
5414
5415     my $name = $dev->{name};
5416
5417     my $fn = "$pcisysfs/devices/$name/reset";
5418
5419     return file_write($fn, "1");
5420 }
5421
5422 sub pci_dev_bind_to_vfio {
5423     my ($dev) = @_;
5424
5425     my $name = $dev->{name};
5426
5427     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5428
5429     if (!-d $vfio_basedir) {
5430         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5431     }
5432     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5433
5434     my $testdir = "$vfio_basedir/$name";
5435     return 1 if -d $testdir;
5436
5437     my $data = "$dev->{vendor} $dev->{product}";
5438     return undef if !file_write("$vfio_basedir/new_id", $data);
5439
5440     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5441     if (!file_write($fn, $name)) {
5442         return undef if -f $fn;
5443     }
5444
5445     $fn = "$vfio_basedir/bind";
5446     if (! -d $testdir) {
5447         return undef if !file_write($fn, $name);
5448     }
5449
5450     return -d $testdir;
5451 }
5452
5453 sub pci_dev_group_bind_to_vfio {
5454     my ($pciid) = @_;
5455
5456     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5457
5458     if (!-d $vfio_basedir) {
5459         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5460     }
5461     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5462
5463     # get IOMMU group devices
5464     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5465       my @devs = grep /^0000:/, readdir($D);
5466     closedir($D);
5467
5468     foreach my $pciid (@devs) {
5469         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5470
5471         # pci bridges, switches or root ports are not supported
5472         # they have a pci_bus subdirectory so skip them
5473         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5474
5475         my $info = pci_device_info($1);
5476         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5477     }
5478
5479     return 1;
5480 }
5481
5482 # vzdump restore implementaion
5483
5484 sub tar_archive_read_firstfile {
5485     my $archive = shift;
5486
5487     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5488
5489     # try to detect archive type first
5490     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5491         die "unable to open file '$archive'\n";
5492     my $firstfile = <$fh>;
5493     kill 15, $pid;
5494     close $fh;
5495
5496     die "ERROR: archive contaions no data\n" if !$firstfile;
5497     chomp $firstfile;
5498
5499     return $firstfile;
5500 }
5501
5502 sub tar_restore_cleanup {
5503     my ($storecfg, $statfile) = @_;
5504
5505     print STDERR "starting cleanup\n";
5506
5507     if (my $fd = IO::File->new($statfile, "r")) {
5508         while (defined(my $line = <$fd>)) {
5509             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5510                 my $volid = $2;
5511                 eval {
5512                     if ($volid =~ m|^/|) {
5513                         unlink $volid || die 'unlink failed\n';
5514                     } else {
5515                         PVE::Storage::vdisk_free($storecfg, $volid);
5516                     }
5517                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5518                 };
5519                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5520             } else {
5521                 print STDERR "unable to parse line in statfile - $line";
5522             }
5523         }
5524         $fd->close();
5525     }
5526 }
5527
5528 sub restore_archive {
5529     my ($archive, $vmid, $user, $opts) = @_;
5530
5531     my $format = $opts->{format};
5532     my $comp;
5533
5534     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5535         $format = 'tar' if !$format;
5536         $comp = 'gzip';
5537     } elsif ($archive =~ m/\.tar$/) {
5538         $format = 'tar' if !$format;
5539     } elsif ($archive =~ m/.tar.lzo$/) {
5540         $format = 'tar' if !$format;
5541         $comp = 'lzop';
5542     } elsif ($archive =~ m/\.vma$/) {
5543         $format = 'vma' if !$format;
5544     } elsif ($archive =~ m/\.vma\.gz$/) {
5545         $format = 'vma' if !$format;
5546         $comp = 'gzip';
5547     } elsif ($archive =~ m/\.vma\.lzo$/) {
5548         $format = 'vma' if !$format;
5549         $comp = 'lzop';
5550     } else {
5551         $format = 'vma' if !$format; # default
5552     }
5553
5554     # try to detect archive format
5555     if ($format eq 'tar') {
5556         return restore_tar_archive($archive, $vmid, $user, $opts);
5557     } else {
5558         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5559     }
5560 }
5561
5562 sub restore_update_config_line {
5563     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5564
5565     return if $line =~ m/^\#qmdump\#/;
5566     return if $line =~ m/^\#vzdump\#/;
5567     return if $line =~ m/^lock:/;
5568     return if $line =~ m/^unused\d+:/;
5569     return if $line =~ m/^parent:/;
5570     return if $line =~ m/^template:/; # restored VM is never a template
5571
5572     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5573     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5574         # try to convert old 1.X settings
5575         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5576         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5577             my ($model, $macaddr) = split(/\=/, $devconfig);
5578             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5579             my $net = {
5580                 model => $model,
5581                 bridge => "vmbr$ind",
5582                 macaddr => $macaddr,
5583             };
5584             my $netstr = print_net($net);
5585
5586             print $outfd "net$cookie->{netcount}: $netstr\n";
5587             $cookie->{netcount}++;
5588         }
5589     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5590         my ($id, $netstr) = ($1, $2);
5591         my $net = parse_net($netstr);
5592         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5593         $netstr = print_net($net);
5594         print $outfd "$id: $netstr\n";
5595     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5596         my $virtdev = $1;
5597         my $value = $3;
5598         my $di = parse_drive($virtdev, $value);
5599         if (defined($di->{backup}) && !$di->{backup}) {
5600             print $outfd "#$line";
5601         } elsif ($map->{$virtdev}) {
5602             delete $di->{format}; # format can change on restore
5603             $di->{file} = $map->{$virtdev};
5604             $value = print_drive($vmid, $di);
5605             print $outfd "$virtdev: $value\n";
5606         } else {
5607             print $outfd $line;
5608         }
5609     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5610         my $vmgenid = $1;
5611         if ($vmgenid ne '0') {
5612             # always generate a new vmgenid if there was a valid one setup
5613             $vmgenid = generate_uuid();
5614         }
5615         print $outfd "vmgenid: $vmgenid\n";
5616     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5617         my ($uuid, $uuid_str);
5618         UUID::generate($uuid);
5619         UUID::unparse($uuid, $uuid_str);
5620         my $smbios1 = parse_smbios1($2);
5621         $smbios1->{uuid} = $uuid_str;
5622         print $outfd $1.print_smbios1($smbios1)."\n";
5623     } else {
5624         print $outfd $line;
5625     }
5626 }
5627
5628 sub scan_volids {
5629     my ($cfg, $vmid) = @_;
5630
5631     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5632
5633     my $volid_hash = {};
5634     foreach my $storeid (keys %$info) {
5635         foreach my $item (@{$info->{$storeid}}) {
5636             next if !($item->{volid} && $item->{size});
5637             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5638             $volid_hash->{$item->{volid}} = $item;
5639         }
5640     }
5641
5642     return $volid_hash;
5643 }
5644
5645 sub is_volume_in_use {
5646     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5647
5648     my $path = PVE::Storage::path($storecfg, $volid);
5649
5650     my $scan_config = sub {
5651         my ($cref, $snapname) = @_;
5652
5653         foreach my $key (keys %$cref) {
5654             my $value = $cref->{$key};
5655             if (is_valid_drivename($key)) {
5656                 next if $skip_drive && $key eq $skip_drive;
5657                 my $drive = parse_drive($key, $value);
5658                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5659                 return 1 if $volid eq $drive->{file};
5660                 if ($drive->{file} =~ m!^/!) {
5661                     return 1 if $drive->{file} eq $path;
5662                 } else {
5663                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5664                     next if !$storeid;
5665                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5666                     next if !$scfg;
5667                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5668                 }
5669             }
5670         }
5671
5672         return 0;
5673     };
5674
5675     return 1 if &$scan_config($conf);
5676
5677     undef $skip_drive;
5678
5679     foreach my $snapname (keys %{$conf->{snapshots}}) {
5680         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5681     }
5682
5683     return 0;
5684 }
5685
5686 sub update_disksize {
5687     my ($vmid, $conf, $volid_hash) = @_;
5688
5689     my $changes;
5690     my $prefix = "VM $vmid:";
5691
5692     # used and unused disks
5693     my $referenced = {};
5694
5695     # Note: it is allowed to define multiple storages with same path (alias), so
5696     # we need to check both 'volid' and real 'path' (two different volid can point
5697     # to the same path).
5698
5699     my $referencedpath = {};
5700
5701     # update size info
5702     foreach my $opt (keys %$conf) {
5703         if (is_valid_drivename($opt)) {
5704             my $drive = parse_drive($opt, $conf->{$opt});
5705             my $volid = $drive->{file};
5706             next if !$volid;
5707
5708             $referenced->{$volid} = 1;
5709             if ($volid_hash->{$volid} &&
5710                 (my $path = $volid_hash->{$volid}->{path})) {
5711                 $referencedpath->{$path} = 1;
5712             }
5713
5714             next if drive_is_cdrom($drive);
5715             next if !$volid_hash->{$volid};
5716
5717             $drive->{size} = $volid_hash->{$volid}->{size};
5718             my $new = print_drive($vmid, $drive);
5719             if ($new ne $conf->{$opt}) {
5720                 $changes = 1;
5721                 $conf->{$opt} = $new;
5722                 print "$prefix update disk '$opt' information.\n";
5723             }
5724         }
5725     }
5726
5727     # remove 'unusedX' entry if volume is used
5728     foreach my $opt (keys %$conf) {
5729         next if $opt !~ m/^unused\d+$/;
5730         my $volid = $conf->{$opt};
5731         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5732         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5733             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5734             $changes = 1;
5735             delete $conf->{$opt};
5736         }
5737
5738         $referenced->{$volid} = 1;
5739         $referencedpath->{$path} = 1 if $path;
5740     }
5741
5742     foreach my $volid (sort keys %$volid_hash) {
5743         next if $volid =~ m/vm-$vmid-state-/;
5744         next if $referenced->{$volid};
5745         my $path = $volid_hash->{$volid}->{path};
5746         next if !$path; # just to be sure
5747         next if $referencedpath->{$path};
5748         $changes = 1;
5749         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5750         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5751         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5752     }
5753
5754     return $changes;
5755 }
5756
5757 sub rescan {
5758     my ($vmid, $nolock, $dryrun) = @_;
5759
5760     my $cfg = PVE::Storage::config();
5761
5762     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5763     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5764     foreach my $stor (keys %{$cfg->{ids}}) {
5765         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5766     }
5767
5768     print "rescan volumes...\n";
5769     my $volid_hash = scan_volids($cfg, $vmid);
5770
5771     my $updatefn =  sub {
5772         my ($vmid) = @_;
5773
5774         my $conf = PVE::QemuConfig->load_config($vmid);
5775
5776         PVE::QemuConfig->check_lock($conf);
5777
5778         my $vm_volids = {};
5779         foreach my $volid (keys %$volid_hash) {
5780             my $info = $volid_hash->{$volid};
5781             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5782         }
5783
5784         my $changes = update_disksize($vmid, $conf, $vm_volids);
5785
5786         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5787     };
5788
5789     if (defined($vmid)) {
5790         if ($nolock) {
5791             &$updatefn($vmid);
5792         } else {
5793             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5794         }
5795     } else {
5796         my $vmlist = config_list();
5797         foreach my $vmid (keys %$vmlist) {
5798             if ($nolock) {
5799                 &$updatefn($vmid);
5800             } else {
5801                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5802             }
5803         }
5804     }
5805 }
5806
5807 sub restore_vma_archive {
5808     my ($archive, $vmid, $user, $opts, $comp) = @_;
5809
5810     my $readfrom = $archive;
5811
5812     my $cfg = PVE::Storage::config();
5813     my $commands = [];
5814     my $bwlimit = $opts->{bwlimit};
5815
5816     my $dbg_cmdstring = '';
5817     my $add_pipe = sub {
5818         my ($cmd) = @_;
5819         push @$commands, $cmd;
5820         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5821         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5822         $readfrom = '-';
5823     };
5824
5825     my $input = undef;
5826     if ($archive eq '-') {
5827         $input = '<&STDIN';
5828     } else {
5829         # If we use a backup from a PVE defined storage we also consider that
5830         # storage's rate limit:
5831         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5832         if (defined($volid)) {
5833             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5834             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5835             if ($readlimit) {
5836                 print STDERR "applying read rate limit: $readlimit\n";
5837                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5838                 $add_pipe->($cstream);
5839             }
5840         }
5841     }
5842
5843     if ($comp) {
5844         my $cmd;
5845         if ($comp eq 'gzip') {
5846             $cmd = ['zcat', $readfrom];
5847         } elsif ($comp eq 'lzop') {
5848             $cmd = ['lzop', '-d', '-c', $readfrom];
5849         } else {
5850             die "unknown compression method '$comp'\n";
5851         }
5852         $add_pipe->($cmd);
5853     }
5854
5855     my $tmpdir = "/var/tmp/vzdumptmp$$";
5856     rmtree $tmpdir;
5857
5858     # disable interrupts (always do cleanups)
5859     local $SIG{INT} =
5860         local $SIG{TERM} =
5861         local $SIG{QUIT} =
5862         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5863
5864     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5865     POSIX::mkfifo($mapfifo, 0600);
5866     my $fifofh;
5867
5868     my $openfifo = sub {
5869         open($fifofh, '>', $mapfifo) || die $!;
5870     };
5871
5872     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5873
5874     my $oldtimeout;
5875     my $timeout = 5;
5876
5877     my $devinfo = {};
5878
5879     my $rpcenv = PVE::RPCEnvironment::get();
5880
5881     my $conffile = PVE::QemuConfig->config_file($vmid);
5882     my $tmpfn = "$conffile.$$.tmp";
5883
5884     # Note: $oldconf is undef if VM does not exists
5885     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5886     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5887
5888     my %storage_limits;
5889
5890     my $print_devmap = sub {
5891         my $virtdev_hash = {};
5892
5893         my $cfgfn = "$tmpdir/qemu-server.conf";
5894
5895         # we can read the config - that is already extracted
5896         my $fh = IO::File->new($cfgfn, "r") ||
5897             "unable to read qemu-server.conf - $!\n";
5898
5899         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5900         if (-f $fwcfgfn) {
5901             my $pve_firewall_dir = '/etc/pve/firewall';
5902             mkdir $pve_firewall_dir; # make sure the dir exists
5903             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5904         }
5905
5906         while (defined(my $line = <$fh>)) {
5907             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5908                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5909                 die "archive does not contain data for drive '$virtdev'\n"
5910                     if !$devinfo->{$devname};
5911                 if (defined($opts->{storage})) {
5912                     $storeid = $opts->{storage} || 'local';
5913                 } elsif (!$storeid) {
5914                     $storeid = 'local';
5915                 }
5916                 $format = 'raw' if !$format;
5917                 $devinfo->{$devname}->{devname} = $devname;
5918                 $devinfo->{$devname}->{virtdev} = $virtdev;
5919                 $devinfo->{$devname}->{format} = $format;
5920                 $devinfo->{$devname}->{storeid} = $storeid;
5921
5922                 # check permission on storage
5923                 my $pool = $opts->{pool}; # todo: do we need that?
5924                 if ($user ne 'root@pam') {
5925                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5926                 }
5927
5928                 $storage_limits{$storeid} = $bwlimit;
5929
5930                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5931             }
5932         }
5933
5934         foreach my $key (keys %storage_limits) {
5935             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5936             next if !$limit;
5937             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5938             $storage_limits{$key} = $limit * 1024;
5939         }
5940
5941         foreach my $devname (keys %$devinfo) {
5942             die "found no device mapping information for device '$devname'\n"
5943                 if !$devinfo->{$devname}->{virtdev};
5944         }
5945
5946         # create empty/temp config
5947         if ($oldconf) {
5948             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5949             foreach_drive($oldconf, sub {
5950                 my ($ds, $drive) = @_;
5951
5952                 return if drive_is_cdrom($drive);
5953
5954                 my $volid = $drive->{file};
5955
5956                 return if !$volid || $volid =~ m|^/|;
5957
5958                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5959                 return if !$path || !$owner || ($owner != $vmid);
5960
5961                 # Note: only delete disk we want to restore
5962                 # other volumes will become unused
5963                 if ($virtdev_hash->{$ds}) {
5964                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5965                     if (my $err = $@) {
5966                         warn $err;
5967                     }
5968                 }
5969             });
5970
5971             # delete vmstate files
5972             # since after the restore we have no snapshots anymore
5973             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5974                 my $snap = $oldconf->{snapshots}->{$snapname};
5975                 if ($snap->{vmstate}) {
5976                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5977                     if (my $err = $@) {
5978                         warn $err;
5979                     }
5980                 }
5981             }
5982         }
5983
5984         my $map = {};
5985         foreach my $virtdev (sort keys %$virtdev_hash) {
5986             my $d = $virtdev_hash->{$virtdev};
5987             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5988             my $storeid = $d->{storeid};
5989             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5990
5991             my $map_opts = '';
5992             if (my $limit = $storage_limits{$storeid}) {
5993                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5994             }
5995
5996             # test if requested format is supported
5997             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5998             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5999             $d->{format} = $defFormat if !$supported;
6000
6001             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
6002                                                   $d->{format}, undef, $alloc_size);
6003             print STDERR "new volume ID is '$volid'\n";
6004             $d->{volid} = $volid;
6005             my $path = PVE::Storage::path($cfg, $volid);
6006
6007             PVE::Storage::activate_volumes($cfg,[$volid]);
6008
6009             my $write_zeros = 1;
6010             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6011                 $write_zeros = 0;
6012             }
6013
6014             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6015
6016             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6017             $map->{$virtdev} = $volid;
6018         }
6019
6020         $fh->seek(0, 0) || die "seek failed - $!\n";
6021
6022         my $outfd = new IO::File ($tmpfn, "w") ||
6023             die "unable to write config for VM $vmid\n";
6024
6025         my $cookie = { netcount => 0 };
6026         while (defined(my $line = <$fh>)) {
6027             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6028         }
6029
6030         $fh->close();
6031         $outfd->close();
6032     };
6033
6034     eval {
6035         # enable interrupts
6036         local $SIG{INT} =
6037             local $SIG{TERM} =
6038             local $SIG{QUIT} =
6039             local $SIG{HUP} =
6040             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6041         local $SIG{ALRM} = sub { die "got timeout\n"; };
6042
6043         $oldtimeout = alarm($timeout);
6044
6045         my $parser = sub {
6046             my $line = shift;
6047
6048             print "$line\n";
6049
6050             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6051                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6052                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6053             } elsif ($line =~ m/^CTIME: /) {
6054                 # we correctly received the vma config, so we can disable
6055                 # the timeout now for disk allocation (set to 10 minutes, so
6056                 # that we always timeout if something goes wrong)
6057                 alarm(600);
6058                 &$print_devmap();
6059                 print $fifofh "done\n";
6060                 my $tmp = $oldtimeout || 0;
6061                 $oldtimeout = undef;
6062                 alarm($tmp);
6063                 close($fifofh);
6064             }
6065         };
6066
6067         print "restore vma archive: $dbg_cmdstring\n";
6068         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6069     };
6070     my $err = $@;
6071
6072     alarm($oldtimeout) if $oldtimeout;
6073
6074     my $vollist = [];
6075     foreach my $devname (keys %$devinfo) {
6076         my $volid = $devinfo->{$devname}->{volid};
6077         push @$vollist, $volid if $volid;
6078     }
6079
6080     PVE::Storage::deactivate_volumes($cfg, $vollist);
6081
6082     unlink $mapfifo;
6083
6084     if ($err) {
6085         rmtree $tmpdir;
6086         unlink $tmpfn;
6087
6088         foreach my $devname (keys %$devinfo) {
6089             my $volid = $devinfo->{$devname}->{volid};
6090             next if !$volid;
6091             eval {
6092                 if ($volid =~ m|^/|) {
6093                     unlink $volid || die 'unlink failed\n';
6094                 } else {
6095                     PVE::Storage::vdisk_free($cfg, $volid);
6096                 }
6097                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6098             };
6099             print STDERR "unable to cleanup '$volid' - $@" if $@;
6100         }
6101         die $err;
6102     }
6103
6104     rmtree $tmpdir;
6105
6106     rename($tmpfn, $conffile) ||
6107         die "unable to commit configuration file '$conffile'\n";
6108
6109     PVE::Cluster::cfs_update(); # make sure we read new file
6110
6111     eval { rescan($vmid, 1); };
6112     warn $@ if $@;
6113 }
6114
6115 sub restore_tar_archive {
6116     my ($archive, $vmid, $user, $opts) = @_;
6117
6118     if ($archive ne '-') {
6119         my $firstfile = tar_archive_read_firstfile($archive);
6120         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6121             if $firstfile ne 'qemu-server.conf';
6122     }
6123
6124     my $storecfg = PVE::Storage::config();
6125
6126     # destroy existing data - keep empty config
6127     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6128     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6129
6130     my $tocmd = "/usr/lib/qemu-server/qmextract";
6131
6132     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6133     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6134     $tocmd .= ' --prealloc' if $opts->{prealloc};
6135     $tocmd .= ' --info' if $opts->{info};
6136
6137     # tar option "xf" does not autodetect compression when read from STDIN,
6138     # so we pipe to zcat
6139     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6140         PVE::Tools::shellquote("--to-command=$tocmd");
6141
6142     my $tmpdir = "/var/tmp/vzdumptmp$$";
6143     mkpath $tmpdir;
6144
6145     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6146     local $ENV{VZDUMP_VMID} = $vmid;
6147     local $ENV{VZDUMP_USER} = $user;
6148
6149     my $conffile = PVE::QemuConfig->config_file($vmid);
6150     my $tmpfn = "$conffile.$$.tmp";
6151
6152     # disable interrupts (always do cleanups)
6153     local $SIG{INT} =
6154         local $SIG{TERM} =
6155         local $SIG{QUIT} =
6156         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6157
6158     eval {
6159         # enable interrupts
6160         local $SIG{INT} =
6161             local $SIG{TERM} =
6162             local $SIG{QUIT} =
6163             local $SIG{HUP} =
6164             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6165
6166         if ($archive eq '-') {
6167             print "extracting archive from STDIN\n";
6168             run_command($cmd, input => "<&STDIN");
6169         } else {
6170             print "extracting archive '$archive'\n";
6171             run_command($cmd);
6172         }
6173
6174         return if $opts->{info};
6175
6176         # read new mapping
6177         my $map = {};
6178         my $statfile = "$tmpdir/qmrestore.stat";
6179         if (my $fd = IO::File->new($statfile, "r")) {
6180             while (defined (my $line = <$fd>)) {
6181                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6182                     $map->{$1} = $2 if $1;
6183                 } else {
6184                     print STDERR "unable to parse line in statfile - $line\n";
6185                 }
6186             }
6187             $fd->close();
6188         }
6189
6190         my $confsrc = "$tmpdir/qemu-server.conf";
6191
6192         my $srcfd = new IO::File($confsrc, "r") ||
6193             die "unable to open file '$confsrc'\n";
6194
6195         my $outfd = new IO::File ($tmpfn, "w") ||
6196             die "unable to write config for VM $vmid\n";
6197
6198         my $cookie = { netcount => 0 };
6199         while (defined (my $line = <$srcfd>)) {
6200             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6201         }
6202
6203         $srcfd->close();
6204         $outfd->close();
6205     };
6206     my $err = $@;
6207
6208     if ($err) {
6209
6210         unlink $tmpfn;
6211
6212         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6213
6214         die $err;
6215     }
6216
6217     rmtree $tmpdir;
6218
6219     rename $tmpfn, $conffile ||
6220         die "unable to commit configuration file '$conffile'\n";
6221
6222     PVE::Cluster::cfs_update(); # make sure we read new file
6223
6224     eval { rescan($vmid, 1); };
6225     warn $@ if $@;
6226 };
6227
6228 sub foreach_storage_used_by_vm {
6229     my ($conf, $func) = @_;
6230
6231     my $sidhash = {};
6232
6233     foreach_drive($conf, sub {
6234         my ($ds, $drive) = @_;
6235         return if drive_is_cdrom($drive);
6236
6237         my $volid = $drive->{file};
6238
6239         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6240         $sidhash->{$sid} = $sid if $sid;
6241     });
6242
6243     foreach my $sid (sort keys %$sidhash) {
6244         &$func($sid);
6245     }
6246 }
6247
6248 sub do_snapshots_with_qemu {
6249     my ($storecfg, $volid) = @_;
6250
6251     my $storage_name = PVE::Storage::parse_volume_id($volid);
6252
6253     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6254         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6255         return 1;
6256     }
6257
6258     if ($volid =~ m/\.(qcow2|qed)$/){
6259         return 1;
6260     }
6261
6262     return undef;
6263 }
6264
6265 sub qga_check_running {
6266     my ($vmid, $nowarn) = @_;
6267
6268     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6269     if ($@) {
6270         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6271         return 0;
6272     }
6273     return 1;
6274 }
6275
6276 sub template_create {
6277     my ($vmid, $conf, $disk) = @_;
6278
6279     my $storecfg = PVE::Storage::config();
6280
6281     foreach_drive($conf, sub {
6282         my ($ds, $drive) = @_;
6283
6284         return if drive_is_cdrom($drive);
6285         return if $disk && $ds ne $disk;
6286
6287         my $volid = $drive->{file};
6288         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6289
6290         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6291         $drive->{file} = $voliddst;
6292         $conf->{$ds} = print_drive($vmid, $drive);
6293         PVE::QemuConfig->write_config($vmid, $conf);
6294     });
6295 }
6296
6297 sub qemu_img_convert {
6298     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6299
6300     my $storecfg = PVE::Storage::config();
6301     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6302     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6303
6304     if ($src_storeid && $dst_storeid) {
6305
6306         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6307
6308         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6309         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6310
6311         my $src_format = qemu_img_format($src_scfg, $src_volname);
6312         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6313
6314         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6315         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6316
6317         my $cmd = [];
6318         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6319         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6320         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6321         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6322         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6323         if ($is_zero_initialized) {
6324             push @$cmd, "zeroinit:$dst_path";
6325         } else {
6326             push @$cmd, $dst_path;
6327         }
6328
6329         my $parser = sub {
6330             my $line = shift;
6331             if($line =~ m/\((\S+)\/100\%\)/){
6332                 my $percent = $1;
6333                 my $transferred = int($size * $percent / 100);
6334                 my $remaining = $size - $transferred;
6335
6336                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6337             }
6338
6339         };
6340
6341         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6342         my $err = $@;
6343         die "copy failed: $err" if $err;
6344     }
6345 }
6346
6347 sub qemu_img_format {
6348     my ($scfg, $volname) = @_;
6349
6350     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6351         return $1;
6352     } else {
6353         return "raw";
6354     }
6355 }
6356
6357 sub qemu_drive_mirror {
6358     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6359
6360     $jobs = {} if !$jobs;
6361
6362     my $qemu_target;
6363     my $format;
6364     $jobs->{"drive-$drive"} = {};
6365
6366     if ($dst_volid =~ /^nbd:/) {
6367         $qemu_target = $dst_volid;
6368         $format = "nbd";
6369     } else {
6370         my $storecfg = PVE::Storage::config();
6371         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6372
6373         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6374
6375         $format = qemu_img_format($dst_scfg, $dst_volname);
6376
6377         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6378
6379         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6380     }
6381
6382     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6383     $opts->{format} = $format if $format;
6384
6385     print "drive mirror is starting for drive-$drive\n";
6386
6387     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6388
6389     if (my $err = $@) {
6390         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6391         die "mirroring error: $err";
6392     }
6393
6394     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6395 }
6396
6397 sub qemu_drive_mirror_monitor {
6398     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6399
6400     eval {
6401         my $err_complete = 0;
6402
6403         while (1) {
6404             die "storage migration timed out\n" if $err_complete > 300;
6405
6406             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6407
6408             my $running_mirror_jobs = {};
6409             foreach my $stat (@$stats) {
6410                 next if $stat->{type} ne 'mirror';
6411                 $running_mirror_jobs->{$stat->{device}} = $stat;
6412             }
6413
6414             my $readycounter = 0;
6415
6416             foreach my $job (keys %$jobs) {
6417
6418                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6419                     print "$job : finished\n";
6420                     delete $jobs->{$job};
6421                     next;
6422                 }
6423
6424                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6425
6426                 my $busy = $running_mirror_jobs->{$job}->{busy};
6427                 my $ready = $running_mirror_jobs->{$job}->{ready};
6428                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6429                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6430                     my $remaining = $total - $transferred;
6431                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6432
6433                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6434                 }
6435
6436                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6437             }
6438
6439             last if scalar(keys %$jobs) == 0;
6440
6441             if ($readycounter == scalar(keys %$jobs)) {
6442                 print "all mirroring jobs are ready \n";
6443                 last if $skipcomplete; #do the complete later
6444
6445                 if ($vmiddst && $vmiddst != $vmid) {
6446                     my $agent_running = $qga && qga_check_running($vmid);
6447                     if ($agent_running) {
6448                         print "freeze filesystem\n";
6449                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6450                     } else {
6451                         print "suspend vm\n";
6452                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6453                     }
6454
6455                     # if we clone a disk for a new target vm, we don't switch the disk
6456                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6457
6458                     if ($agent_running) {
6459                         print "unfreeze filesystem\n";
6460                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6461                     } else {
6462                         print "resume vm\n";
6463                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6464                     }
6465
6466                     last;
6467                 } else {
6468
6469                     foreach my $job (keys %$jobs) {
6470                         # try to switch the disk if source and destination are on the same guest
6471                         print "$job: Completing block job...\n";
6472
6473                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6474                         if ($@ =~ m/cannot be completed/) {
6475                             print "$job: Block job cannot be completed, try again.\n";
6476                             $err_complete++;
6477                         }else {
6478                             print "$job: Completed successfully.\n";
6479                             $jobs->{$job}->{complete} = 1;
6480                         }
6481                     }
6482                 }
6483             }
6484             sleep 1;
6485         }
6486     };
6487     my $err = $@;
6488
6489     if ($err) {
6490         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6491         die "mirroring error: $err";
6492     }
6493
6494 }
6495
6496 sub qemu_blockjobs_cancel {
6497     my ($vmid, $jobs) = @_;
6498
6499     foreach my $job (keys %$jobs) {
6500         print "$job: Cancelling block job\n";
6501         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6502         $jobs->{$job}->{cancel} = 1;
6503     }
6504
6505     while (1) {
6506         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6507
6508         my $running_jobs = {};
6509         foreach my $stat (@$stats) {
6510             $running_jobs->{$stat->{device}} = $stat;
6511         }
6512
6513         foreach my $job (keys %$jobs) {
6514
6515             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6516                 print "$job: Done.\n";
6517                 delete $jobs->{$job};
6518             }
6519         }
6520
6521         last if scalar(keys %$jobs) == 0;
6522
6523         sleep 1;
6524     }
6525 }
6526
6527 sub clone_disk {
6528     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6529         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6530
6531     my $newvolid;
6532
6533     if (!$full) {
6534         print "create linked clone of drive $drivename ($drive->{file})\n";
6535         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6536         push @$newvollist, $newvolid;
6537     } else {
6538
6539         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6540         $storeid = $storage if $storage;
6541
6542         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6543         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6544
6545         print "create full clone of drive $drivename ($drive->{file})\n";
6546         my $name = undef;
6547         if (drive_is_cloudinit($drive)) {
6548             $name = "vm-$newvmid-cloudinit";
6549             # cloudinit only supports raw and qcow2 atm:
6550             if ($dst_format eq 'qcow2') {
6551                 $name .= '.qcow2';
6552             } elsif ($dst_format ne 'raw') {
6553                 die "clone: unhandled format for cloudinit image\n";
6554             }
6555         }
6556         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6557         push @$newvollist, $newvolid;
6558
6559         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6560
6561         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6562         if (!$running || $snapname) {
6563             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6564         } else {
6565
6566             my $kvmver = get_running_qemu_version ($vmid);
6567             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6568                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6569                     if $drive->{iothread};
6570             }
6571
6572             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6573         }
6574     }
6575
6576     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6577
6578     my $disk = $drive;
6579     $disk->{format} = undef;
6580     $disk->{file} = $newvolid;
6581     $disk->{size} = $size;
6582
6583     return $disk;
6584 }
6585
6586 # this only works if VM is running
6587 sub get_current_qemu_machine {
6588     my ($vmid) = @_;
6589
6590     my $cmd = { execute => 'query-machines', arguments => {} };
6591     my $res = vm_qmp_command($vmid, $cmd);
6592
6593     my ($current, $default);
6594     foreach my $e (@$res) {
6595         $default = $e->{name} if $e->{'is-default'};
6596         $current = $e->{name} if $e->{'is-current'};
6597     }
6598
6599     # fallback to the default machine if current is not supported by qemu
6600     return $current || $default || 'pc';
6601 }
6602
6603 sub get_running_qemu_version {
6604     my ($vmid) = @_;
6605     my $cmd = { execute => 'query-version', arguments => {} };
6606     my $res = vm_qmp_command($vmid, $cmd);
6607     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6608 }
6609
6610 sub qemu_machine_feature_enabled {
6611     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6612
6613     my $current_major;
6614     my $current_minor;
6615
6616     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6617
6618         $current_major = $3;
6619         $current_minor = $4;
6620
6621     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6622
6623         $current_major = $1;
6624         $current_minor = $2;
6625     }
6626
6627     return 1 if $current_major > $version_major ||
6628                 ($current_major == $version_major &&
6629                  $current_minor >= $version_minor);
6630 }
6631
6632 sub qemu_machine_pxe {
6633     my ($vmid, $conf, $machine) = @_;
6634
6635     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6636
6637     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6638         $machine .= '.pxe';
6639     }
6640
6641     return $machine;
6642 }
6643
6644 sub qemu_use_old_bios_files {
6645     my ($machine_type) = @_;
6646
6647     return if !$machine_type;
6648
6649     my $use_old_bios_files = undef;
6650
6651     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6652         $machine_type = $1;
6653         $use_old_bios_files = 1;
6654     } else {
6655         my $kvmver = kvm_user_version();
6656         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6657         # load new efi bios files on migration. So this hack is required to allow
6658         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6659         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6660         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6661     }
6662
6663     return ($use_old_bios_files, $machine_type);
6664 }
6665
6666 sub create_efidisk {
6667     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6668
6669     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6670
6671     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6672     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6673     PVE::Storage::activate_volumes($storecfg, [$volid]);
6674
6675     my $path = PVE::Storage::path($storecfg, $volid);
6676     eval {
6677         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6678     };
6679     die "Copying EFI vars image failed: $@" if $@;
6680
6681     return ($volid, $vars_size);
6682 }
6683
6684 sub lspci {
6685
6686     my $devices = {};
6687
6688     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6689             my (undef, $id, $function) = @_;
6690             my $res = { id => $id, function => $function};
6691             push @{$devices->{$id}}, $res;
6692     });
6693
6694     # Entries should be sorted by functions.
6695     foreach my $id (keys %$devices) {
6696         my $dev = $devices->{$id};
6697         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6698     }
6699
6700     return $devices;
6701 }
6702
6703 sub vm_iothreads_list {
6704     my ($vmid) = @_;
6705
6706     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6707
6708     my $iothreads = {};
6709     foreach my $iothread (@$res) {
6710         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6711     }
6712
6713     return $iothreads;
6714 }
6715
6716 sub scsihw_infos {
6717     my ($conf, $drive) = @_;
6718
6719     my $maxdev = 0;
6720
6721     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6722         $maxdev = 7;
6723     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6724         $maxdev = 1;
6725     } else {
6726         $maxdev = 256;
6727     }
6728
6729     my $controller = int($drive->{index} / $maxdev);
6730     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6731
6732     return ($maxdev, $controller, $controller_prefix);
6733 }
6734
6735 sub add_hyperv_enlightenments {
6736     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6737
6738     return if $winversion < 6;
6739     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6740
6741     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6742
6743     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6744         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6745         push @$cpuFlags , 'hv_vapic';
6746         push @$cpuFlags , 'hv_time';
6747     } else {
6748         push @$cpuFlags , 'hv_spinlocks=0xffff';
6749     }
6750
6751     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6752         push @$cpuFlags , 'hv_reset';
6753         push @$cpuFlags , 'hv_vpindex';
6754         push @$cpuFlags , 'hv_runtime';
6755     }
6756
6757     if ($winversion >= 7) {
6758         push @$cpuFlags , 'hv_relaxed';
6759
6760         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6761             push @$cpuFlags , 'hv_synic';
6762             push @$cpuFlags , 'hv_stimer';
6763         }
6764     }
6765 }
6766
6767 sub windows_version {
6768     my ($ostype) = @_;
6769
6770     return 0 if !$ostype;
6771
6772     my $winversion = 0;
6773
6774     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6775         $winversion = 5;
6776     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6777         $winversion = 6;
6778     } elsif ($ostype =~ m/^win(\d+)$/) {
6779         $winversion = $1;
6780     }
6781
6782     return $winversion;
6783 }
6784
6785 sub resolve_dst_disk_format {
6786         my ($storecfg, $storeid, $src_volname, $format) = @_;
6787         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6788
6789         if (!$format) {
6790             # if no target format is specified, use the source disk format as hint
6791             if ($src_volname) {
6792                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6793                 $format = qemu_img_format($scfg, $src_volname);
6794             } else {
6795                 return $defFormat;
6796             }
6797         }
6798
6799         # test if requested format is supported - else use default
6800         my $supported = grep { $_ eq $format } @$validFormats;
6801         $format = $defFormat if !$supported;
6802         return $format;
6803 }
6804
6805 sub resolve_first_disk {
6806     my $conf = shift;
6807     my @disks = PVE::QemuServer::valid_drive_names();
6808     my $firstdisk;
6809     foreach my $ds (reverse @disks) {
6810         next if !$conf->{$ds};
6811         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6812         next if PVE::QemuServer::drive_is_cdrom($disk);
6813         $firstdisk = $ds;
6814     }
6815     return $firstdisk;
6816 }
6817
6818 sub generate_uuid {
6819     my ($uuid, $uuid_str);
6820     UUID::generate($uuid);
6821     UUID::unparse($uuid, $uuid_str);
6822     return $uuid_str;
6823 }
6824
6825 sub generate_smbios1_uuid {
6826     return "uuid=".generate_uuid();
6827 }
6828
6829 # bash completion helper
6830
6831 sub complete_backup_archives {
6832     my ($cmdname, $pname, $cvalue) = @_;
6833
6834     my $cfg = PVE::Storage::config();
6835
6836     my $storeid;
6837
6838     if ($cvalue =~ m/^([^:]+):/) {
6839         $storeid = $1;
6840     }
6841
6842     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6843
6844     my $res = [];
6845     foreach my $id (keys %$data) {
6846         foreach my $item (@{$data->{$id}}) {
6847             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6848             push @$res, $item->{volid} if defined($item->{volid});
6849         }
6850     }
6851
6852     return $res;
6853 }
6854
6855 my $complete_vmid_full = sub {
6856     my ($running) = @_;
6857
6858     my $idlist = vmstatus();
6859
6860     my $res = [];
6861
6862     foreach my $id (keys %$idlist) {
6863         my $d = $idlist->{$id};
6864         if (defined($running)) {
6865             next if $d->{template};
6866             next if $running && $d->{status} ne 'running';
6867             next if !$running && $d->{status} eq 'running';
6868         }
6869         push @$res, $id;
6870
6871     }
6872     return $res;
6873 };
6874
6875 sub complete_vmid {
6876     return &$complete_vmid_full();
6877 }
6878
6879 sub complete_vmid_stopped {
6880     return &$complete_vmid_full(0);
6881 }
6882
6883 sub complete_vmid_running {
6884     return &$complete_vmid_full(1);
6885 }
6886
6887 sub complete_storage {
6888
6889     my $cfg = PVE::Storage::config();
6890     my $ids = $cfg->{ids};
6891
6892     my $res = [];
6893     foreach my $sid (keys %$ids) {
6894         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6895         next if !$ids->{$sid}->{content}->{images};
6896         push @$res, $sid;
6897     }
6898
6899     return $res;
6900 }
6901
6902 sub nbd_stop {
6903     my ($vmid) = @_;
6904
6905     vm_mon_cmd($vmid, 'nbd-server-stop');
6906 }
6907
6908 1;