PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  84         description => "Specifies the Qemu machine type.",
  85         type => 'string',
  86         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
  87         maxLength => 40,
  88         optional => 1,
  89 });
  90
  91 #no warnings 'redefine';
  92
  93 sub cgroups_write {
  94    my ($controller, $vmid, $option, $value) = @_;
  95
  96    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  97    PVE::ProcFSTools::write_proc_entry($path, $value);
  98
  99 }
 100
 101 my $nodename = PVE::INotify::nodename();
 102
 103 mkdir "/etc/pve/nodes/$nodename";
 104 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 105 mkdir $confdir;
 106
 107 my $var_run_tmpdir = "/var/run/qemu-server";
 108 mkdir $var_run_tmpdir;
 109
 110 my $lock_dir = "/var/lock/qemu-server";
 111 mkdir $lock_dir;
 112
 113 my $pcisysfs = "/sys/bus/pci";
 114
 115 my $cpu_vendor_list = {
 116     # Intel CPUs
 117     486 => 'GenuineIntel',
 118     pentium => 'GenuineIntel',
 119     pentium2  => 'GenuineIntel',
 120     pentium3  => 'GenuineIntel',
 121     coreduo => 'GenuineIntel',
 122     core2duo => 'GenuineIntel',
 123     Conroe  => 'GenuineIntel',
 124     Penryn  => 'GenuineIntel',
 125     Nehalem  => 'GenuineIntel',
 126     'Nehalem-IBRS'  => 'GenuineIntel',
 127     Westmere => 'GenuineIntel',
 128     'Westmere-IBRS' => 'GenuineIntel',
 129     SandyBridge => 'GenuineIntel',
 130     'SandyBridge-IBRS' => 'GenuineIntel',
 131     IvyBridge => 'GenuineIntel',
 132     'IvyBridge-IBRS' => 'GenuineIntel',
 133     Haswell => 'GenuineIntel',
 134     'Haswell-IBRS' => 'GenuineIntel',
 135     'Haswell-noTSX' => 'GenuineIntel',
 136     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 137     Broadwell => 'GenuineIntel',
 138     'Broadwell-IBRS' => 'GenuineIntel',
 139     'Broadwell-noTSX' => 'GenuineIntel',
 140     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 141     'Skylake-Client' => 'GenuineIntel',
 142     'Skylake-Client-IBRS' => 'GenuineIntel',
 143     'Skylake-Server' => 'GenuineIntel',
 144     'Skylake-Server-IBRS' => 'GenuineIntel',
 145
 146     # AMD CPUs
 147     athlon => 'AuthenticAMD',
 148     phenom  => 'AuthenticAMD',
 149     Opteron_G1  => 'AuthenticAMD',
 150     Opteron_G2  => 'AuthenticAMD',
 151     Opteron_G3  => 'AuthenticAMD',
 152     Opteron_G4  => 'AuthenticAMD',
 153     Opteron_G5  => 'AuthenticAMD',
 154     EPYC => 'AuthenticAMD',
 155     'EPYC-IBPB' => 'AuthenticAMD',
 156
 157     # generic types, use vendor from host node
 158     host => 'default',
 159     kvm32 => 'default',
 160     kvm64 => 'default',
 161     qemu32 => 'default',
 162     qemu64 => 'default',
 163     max => 'default',
 164 };
 165
 166 my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
 167
 168 my $cpu_fmt = {
 169     cputype => {
 170         description => "Emulated CPU type.",
 171         type => 'string',
 172         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 173         default => 'kvm64',
 174         default_key => 1,
 175     },
 176     hidden => {
 177         description => "Do not identify as a KVM virtual machine.",
 178         type => 'boolean',
 179         optional => 1,
 180         default => 0
 181     },
 182     flags => {
 183         description => "List of additional CPU flags separated by ';'."
 184                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 185                      . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
 186         format_description => '+FLAG[;-FLAG...]',
 187         type => 'string',
 188         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 189         optional => 1,
 190     },
 191 };
 192
 193 my $watchdog_fmt = {
 194     model => {
 195         default_key => 1,
 196         type => 'string',
 197         enum => [qw(i6300esb ib700)],
 198         description => "Watchdog type to emulate.",
 199         default => 'i6300esb',
 200         optional => 1,
 201     },
 202     action => {
 203         type => 'string',
 204         enum => [qw(reset shutdown poweroff pause debug none)],
 205         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 206         optional => 1,
 207     },
 208 };
 209 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 210
 211 my $agent_fmt = {
 212     enabled => {
 213         description => "Enable/disable Qemu GuestAgent.",
 214         type => 'boolean',
 215         default => 0,
 216         default_key => 1,
 217     },
 218     fstrim_cloned_disks => {
 219         description => "Run fstrim after cloning/moving a disk.",
 220         type => 'boolean',
 221         optional => 1,
 222         default => 0
 223     },
 224 };
 225
 226 my $confdesc = {
 227     onboot => {
 228         optional => 1,
 229         type => 'boolean',
 230         description => "Specifies whether a VM will be started during system bootup.",
 231         default => 0,
 232     },
 233     autostart => {
 234         optional => 1,
 235         type => 'boolean',
 236         description => "Automatic restart after crash (currently ignored).",
 237         default => 0,
 238     },
 239     hotplug => {
 240         optional => 1,
 241         type => 'string', format => 'pve-hotplug-features',
 242         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 243         default => 'network,disk,usb',
 244     },
 245     reboot => {
 246         optional => 1,
 247         type => 'boolean',
 248         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 249         default => 1,
 250     },
 251     lock => {
 252         optional => 1,
 253         type => 'string',
 254         description => "Lock/unlock the VM.",
 255         enum => [qw(migrate backup snapshot rollback)],
 256     },
 257     cpulimit => {
 258         optional => 1,
 259         type => 'number',
 260         description => "Limit of CPU usage.",
 261         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 262         minimum => 0,
 263         maximum => 128,
 264         default => 0,
 265     },
 266     cpuunits => {
 267         optional => 1,
 268         type => 'integer',
 269         description => "CPU weight for a VM.",
 270         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 271         minimum => 2,
 272         maximum => 262144,
 273         default => 1024,
 274     },
 275     memory => {
 276         optional => 1,
 277         type => 'integer',
 278         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 279         minimum => 16,
 280         default => 512,
 281     },
 282     balloon => {
 283         optional => 1,
 284         type => 'integer',
 285         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 286         minimum => 0,
 287     },
 288     shares => {
 289         optional => 1,
 290         type => 'integer',
 291         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 292         minimum => 0,
 293         maximum => 50000,
 294         default => 1000,
 295     },
 296     keyboard => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 300                        "It should not be necessary to set it.",
 301         enum => PVE::Tools::kvmkeymaplist(),
 302         default => undef,
 303     },
 304     name => {
 305         optional => 1,
 306         type => 'string', format => 'dns-name',
 307         description => "Set a name for the VM. Only used on the configuration web interface.",
 308     },
 309     scsihw => {
 310         optional => 1,
 311         type => 'string',
 312         description => "SCSI controller model",
 313         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 314         default => 'lsi',
 315     },
 316     description => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 320     },
 321     ostype => {
 322         optional => 1,
 323         type => 'string',
 324         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 325         description => "Specify guest operating system.",
 326         verbose_description => <<EODESC,
 327 Specify guest operating system. This is used to enable special
 328 optimization/features for specific operating systems:
 329
 330 [horizontal]
 331 other;; unspecified OS
 332 wxp;; Microsoft Windows XP
 333 w2k;; Microsoft Windows 2000
 334 w2k3;; Microsoft Windows 2003
 335 w2k8;; Microsoft Windows 2008
 336 wvista;; Microsoft Windows Vista
 337 win7;; Microsoft Windows 7
 338 win8;; Microsoft Windows 8/2012/2012r2
 339 win10;; Microsoft Windows 10/2016
 340 l24;; Linux 2.4 Kernel
 341 l26;; Linux 2.6/3.X Kernel
 342 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 343 EODESC
 344     },
 345     boot => {
 346         optional => 1,
 347         type => 'string',
 348         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 349         pattern => '[acdn]{1,4}',
 350         default => 'cdn',
 351     },
 352     bootdisk => {
 353         optional => 1,
 354         type => 'string', format => 'pve-qm-bootdisk',
 355         description => "Enable booting from specified disk.",
 356         pattern => '(ide|sata|scsi|virtio)\d+',
 357     },
 358     smp => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "The number of CPUs. Please use option -sockets instead.",
 362         minimum => 1,
 363         default => 1,
 364     },
 365     sockets => {
 366         optional => 1,
 367         type => 'integer',
 368         description => "The number of CPU sockets.",
 369         minimum => 1,
 370         default => 1,
 371     },
 372     cores => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "The number of cores per socket.",
 376         minimum => 1,
 377         default => 1,
 378     },
 379     numa => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable NUMA.",
 383         default => 0,
 384     },
 385     hugepages => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Enable/disable hugepages memory.",
 389         enum => [qw(any 2 1024)],
 390     },
 391     vcpus => {
 392         optional => 1,
 393         type => 'integer',
 394         description => "Number of hotplugged vcpus.",
 395         minimum => 1,
 396         default => 0,
 397     },
 398     acpi => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Enable/disable ACPI.",
 402         default => 1,
 403     },
 404     agent => {
 405         optional => 1,
 406         description => "Enable/disable Qemu GuestAgent and its properties.",
 407         type => 'string',
 408         format => $agent_fmt,
 409     },
 410     kvm => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable KVM hardware virtualization.",
 414         default => 1,
 415     },
 416     tdf => {
 417         optional => 1,
 418         type => 'boolean',
 419         description => "Enable/disable time drift fix.",
 420         default => 0,
 421     },
 422     localtime => {
 423         optional => 1,
 424         type => 'boolean',
 425         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 426     },
 427     freeze => {
 428         optional => 1,
 429         type => 'boolean',
 430         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 431     },
 432     vga => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Select the VGA type.",
 436         verbose_description => "Select the VGA type. If you want to use high resolution" .
 437             " modes (>= 1280x1024x16) then you should use the options " .
 438             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 439             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 440             "display sever. For win* OS you can select how many independent " .
 441             "displays you want, Linux guests can add displays them self. " .
 442             "You can also run without any graphic card, using a serial device" .
 443             " as terminal.",
 444         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
 445     },
 446     watchdog => {
 447         optional => 1,
 448         type => 'string', format => 'pve-qm-watchdog',
 449         description => "Create a virtual hardware watchdog device.",
 450         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 451             " (by a guest action), the watchdog must be periodically polled " .
 452             "by an agent inside the guest or else the watchdog will reset " .
 453             "the guest (or execute the respective action specified)",
 454     },
 455     startdate => {
 456         optional => 1,
 457         type => 'string',
 458         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 459         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 460         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 461         default => 'now',
 462     },
 463     startup =>  get_standard_option('pve-startup-order'),
 464     template => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable Template.",
 468         default => 0,
 469     },
 470     args => {
 471         optional => 1,
 472         type => 'string',
 473         description => "Arbitrary arguments passed to kvm.",
 474         verbose_description => <<EODESCR,
 475 Arbitrary arguments passed to kvm, for example:
 476
 477 args: -no-reboot -no-hpet
 478
 479 NOTE: this option is for experts only.
 480 EODESCR
 481     },
 482     tablet => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 1,
 486         description => "Enable/disable the USB tablet device.",
 487         verbose_description => "Enable/disable the USB tablet device. This device is " .
 488             "usually needed to allow absolute mouse positioning with VNC. " .
 489             "Else the mouse runs out of sync with normal VNC clients. " .
 490             "If you're running lots of console-only guests on one host, " .
 491             "you may consider disabling this to save some context switches. " .
 492             "This is turned off by default if you use spice (-vga=qxl).",
 493     },
 494     migrate_speed => {
 495         optional => 1,
 496         type => 'integer',
 497         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 498         minimum => 0,
 499         default => 0,
 500     },
 501     migrate_downtime => {
 502         optional => 1,
 503         type => 'number',
 504         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 505         minimum => 0,
 506         default => 0.1,
 507     },
 508     cdrom => {
 509         optional => 1,
 510         type => 'string', format => 'pve-qm-ide',
 511         typetext => '<volume>',
 512         description => "This is an alias for option -ide2",
 513     },
 514     cpu => {
 515         optional => 1,
 516         description => "Emulated CPU type.",
 517         type => 'string',
 518         format => $cpu_fmt,
 519     },
 520     parent => get_standard_option('pve-snapshot-name', {
 521         optional => 1,
 522         description => "Parent snapshot name. This is used internally, and should not be modified.",
 523     }),
 524     snaptime => {
 525         optional => 1,
 526         description => "Timestamp for snapshots.",
 527         type => 'integer',
 528         minimum => 0,
 529     },
 530     vmstate => {
 531         optional => 1,
 532         type => 'string', format => 'pve-volume-id',
 533         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 534     },
 535     vmstatestorage => get_standard_option('pve-storage-id', {
 536         description => "Default storage for VM state volumes/files.",
 537         optional => 1,
 538     }),
 539     runningmachine => get_standard_option('pve-qemu-machine', {
 540         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 541     }),
 542     machine => get_standard_option('pve-qemu-machine'),
 543     smbios1 => {
 544         description => "Specify SMBIOS type 1 fields.",
 545         type => 'string', format => 'pve-qm-smbios1',
 546         maxLength => 256,
 547         optional => 1,
 548     },
 549     protection => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 553         default => 0,
 554     },
 555     bios => {
 556         optional => 1,
 557         type => 'string',
 558         enum => [ qw(seabios ovmf) ],
 559         description => "Select BIOS implementation.",
 560         default => 'seabios',
 561     },
 562     vmgenid => {
 563         type => 'string',
 564         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 565         format_description => 'UUID',
 566         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 567         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 568             " 128-bit integer value identifier to the guest OS. This allows to".
 569             " notify the guest operating system when the virtual machine is".
 570             " executed with a different configuration (e.g. snapshot execution".
 571             " or creation from a template). The guest operating system notices".
 572             " the change, and is then able to react as appropriate by marking".
 573             " its copies of distributed databases as dirty, re-initializing its".
 574             " random number generator, etc.\n".
 575             "Note that auto-creation only works when done throug API/CLI create".
 576             " or update methods, but not when manually editing the config file.",
 577         default => "1 (autogenerated)",
 578         optional => 1,
 579     },
 580 };
 581
 582 my $confdesc_cloudinit = {
 583     citype => {
 584         optional => 1,
 585         type => 'string',
 586         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 587         enum => ['configdrive2', 'nocloud'],
 588     },
 589     ciuser => {
 590         optional => 1,
 591         type => 'string',
 592         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 593     },
 594     cipassword => {
 595         optional => 1,
 596         type => 'string',
 597         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 598     },
 599     searchdomain => {
 600         optional => 1,
 601         type => 'string',
 602         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 603     },
 604     nameserver => {
 605         optional => 1,
 606         type => 'string', format => 'address-list',
 607         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 608     },
 609     sshkeys => {
 610         optional => 1,
 611         type => 'string',
 612         format => 'urlencoded',
 613         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 614     },
 615 };
 616
 617 # what about other qemu settings ?
 618 #cpu => 'string',
 619 #machine => 'string',
 620 #fda => 'file',
 621 #fdb => 'file',
 622 #mtdblock => 'file',
 623 #sd => 'file',
 624 #pflash => 'file',
 625 #snapshot => 'bool',
 626 #bootp => 'file',
 627 ##tftp => 'dir',
 628 ##smb => 'dir',
 629 #kernel => 'file',
 630 #append => 'string',
 631 #initrd => 'file',
 632 ##soundhw => 'string',
 633
 634 while (my ($k, $v) = each %$confdesc) {
 635     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 636 }
 637
 638 my $MAX_IDE_DISKS = 4;
 639 my $MAX_SCSI_DISKS = 14;
 640 my $MAX_VIRTIO_DISKS = 16;
 641 my $MAX_SATA_DISKS = 6;
 642 my $MAX_USB_DEVICES = 5;
 643 my $MAX_NETS = 32;
 644 my $MAX_UNUSED_DISKS = 8;
 645 my $MAX_HOSTPCI_DEVICES = 4;
 646 my $MAX_SERIAL_PORTS = 4;
 647 my $MAX_PARALLEL_PORTS = 3;
 648 my $MAX_NUMA = 8;
 649
 650 my $numa_fmt = {
 651     cpus => {
 652         type => "string",
 653         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 654         description => "CPUs accessing this NUMA node.",
 655         format_description => "id[-id];...",
 656     },
 657     memory => {
 658         type => "number",
 659         description => "Amount of memory this NUMA node provides.",
 660         optional => 1,
 661     },
 662     hostnodes => {
 663         type => "string",
 664         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 665         description => "Host NUMA nodes to use.",
 666         format_description => "id[-id];...",
 667         optional => 1,
 668     },
 669     policy => {
 670         type => 'string',
 671         enum => [qw(preferred bind interleave)],
 672         description => "NUMA allocation policy.",
 673         optional => 1,
 674     },
 675 };
 676 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 677 my $numadesc = {
 678     optional => 1,
 679     type => 'string', format => $numa_fmt,
 680     description => "NUMA topology.",
 681 };
 682 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 683
 684 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 685     $confdesc->{"numa$i"} = $numadesc;
 686 }
 687
 688 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 689                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 690                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 691 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 692
 693 my $net_fmt_bridge_descr = <<__EOD__;
 694 Bridge to attach the network device to. The Proxmox VE standard bridge
 695 is called 'vmbr0'.
 696
 697 If you do not specify a bridge, we create a kvm user (NATed) network
 698 device, which provides DHCP and DNS services. The following addresses
 699 are used:
 700
 701  10.0.2.2   Gateway
 702  10.0.2.3   DNS Server
 703  10.0.2.4   SMB Server
 704
 705 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 706 __EOD__
 707
 708 my $net_fmt = {
 709     macaddr => {
 710         type => 'string',
 711         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 712         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 713         format_description => "XX:XX:XX:XX:XX:XX",
 714         optional => 1,
 715     },
 716     model => {
 717         type => 'string',
 718         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 719         enum => $nic_model_list,
 720         default_key => 1,
 721     },
 722     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 723     bridge => {
 724         type => 'string',
 725         description => $net_fmt_bridge_descr,
 726         format_description => 'bridge',
 727         optional => 1,
 728     },
 729     queues => {
 730         type => 'integer',
 731         minimum => 0, maximum => 16,
 732         description => 'Number of packet queues to be used on the device.',
 733         optional => 1,
 734     },
 735     rate => {
 736         type => 'number',
 737         minimum => 0,
 738         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 739         optional => 1,
 740     },
 741     tag => {
 742         type => 'integer',
 743         minimum => 1, maximum => 4094,
 744         description => 'VLAN tag to apply to packets on this interface.',
 745         optional => 1,
 746     },
 747     trunks => {
 748         type => 'string',
 749         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 750         description => 'VLAN trunks to pass through this interface.',
 751         format_description => 'vlanid[;vlanid...]',
 752         optional => 1,
 753     },
 754     firewall => {
 755         type => 'boolean',
 756         description => 'Whether this interface should be protected by the firewall.',
 757         optional => 1,
 758     },
 759     link_down => {
 760         type => 'boolean',
 761         description => 'Whether this interface should be disconnected (like pulling the plug).',
 762         optional => 1,
 763     },
 764 };
 765
 766 my $netdesc = {
 767     optional => 1,
 768     type => 'string', format => $net_fmt,
 769     description => "Specify network devices.",
 770 };
 771
 772 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 773
 774 my $ipconfig_fmt = {
 775     ip => {
 776         type => 'string',
 777         format => 'pve-ipv4-config',
 778         format_description => 'IPv4Format/CIDR',
 779         description => 'IPv4 address in CIDR format.',
 780         optional => 1,
 781         default => 'dhcp',
 782     },
 783     gw => {
 784         type => 'string',
 785         format => 'ipv4',
 786         format_description => 'GatewayIPv4',
 787         description => 'Default gateway for IPv4 traffic.',
 788         optional => 1,
 789         requires => 'ip',
 790     },
 791     ip6 => {
 792         type => 'string',
 793         format => 'pve-ipv6-config',
 794         format_description => 'IPv6Format/CIDR',
 795         description => 'IPv6 address in CIDR format.',
 796         optional => 1,
 797         default => 'dhcp',
 798     },
 799     gw6 => {
 800         type => 'string',
 801         format => 'ipv6',
 802         format_description => 'GatewayIPv6',
 803         description => 'Default gateway for IPv6 traffic.',
 804         optional => 1,
 805         requires => 'ip6',
 806     },
 807 };
 808 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 809 my $ipconfigdesc = {
 810     optional => 1,
 811     type => 'string', format => 'pve-qm-ipconfig',
 812     description => <<'EODESCR',
 813 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 814
 815 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 816
 817 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 818 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 819
 820 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 821 EODESCR
 822 };
 823 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 824
 825 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 826     $confdesc->{"net$i"} = $netdesc;
 827     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 828 }
 829
 830 foreach my $key (keys %$confdesc_cloudinit) {
 831     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 832 }
 833
 834 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 835 sub verify_volume_id_or_qm_path {
 836     my ($volid, $noerr) = @_;
 837
 838     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 839         return $volid;
 840     }
 841
 842     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 843     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 844     if ($@) {
 845         return undef if $noerr;
 846         die $@;
 847     }
 848     return $volid;
 849 }
 850
 851 my $drivename_hash;
 852
 853 my %drivedesc_base = (
 854     volume => { alias => 'file' },
 855     file => {
 856         type => 'string',
 857         format => 'pve-volume-id-or-qm-path',
 858         default_key => 1,
 859         format_description => 'volume',
 860         description => "The drive's backing volume.",
 861     },
 862     media => {
 863         type => 'string',
 864         enum => [qw(cdrom disk)],
 865         description => "The drive's media type.",
 866         default => 'disk',
 867         optional => 1
 868     },
 869     cyls => {
 870         type => 'integer',
 871         description => "Force the drive's physical geometry to have a specific cylinder count.",
 872         optional => 1
 873     },
 874     heads => {
 875         type => 'integer',
 876         description => "Force the drive's physical geometry to have a specific head count.",
 877         optional => 1
 878     },
 879     secs => {
 880         type => 'integer',
 881         description => "Force the drive's physical geometry to have a specific sector count.",
 882         optional => 1
 883     },
 884     trans => {
 885         type => 'string',
 886         enum => [qw(none lba auto)],
 887         description => "Force disk geometry bios translation mode.",
 888         optional => 1,
 889     },
 890     snapshot => {
 891         type => 'boolean',
 892         description => "Controls qemu's snapshot mode feature."
 893             . " If activated, changes made to the disk are temporary and will"
 894             . " be discarded when the VM is shutdown.",
 895         optional => 1,
 896     },
 897     cache => {
 898         type => 'string',
 899         enum => [qw(none writethrough writeback unsafe directsync)],
 900         description => "The drive's cache mode",
 901         optional => 1,
 902     },
 903     format => get_standard_option('pve-qm-image-format'),
 904     size => {
 905         type => 'string',
 906         format => 'disk-size',
 907         format_description => 'DiskSize',
 908         description => "Disk size. This is purely informational and has no effect.",
 909         optional => 1,
 910     },
 911     backup => {
 912         type => 'boolean',
 913         description => "Whether the drive should be included when making backups.",
 914         optional => 1,
 915     },
 916     replicate => {
 917         type => 'boolean',
 918         description => 'Whether the drive should considered for replication jobs.',
 919         optional => 1,
 920         default => 1,
 921     },
 922     rerror => {
 923         type => 'string',
 924         enum => [qw(ignore report stop)],
 925         description => 'Read error action.',
 926         optional => 1,
 927     },
 928     werror => {
 929         type => 'string',
 930         enum => [qw(enospc ignore report stop)],
 931         description => 'Write error action.',
 932         optional => 1,
 933     },
 934     aio => {
 935         type => 'string',
 936         enum => [qw(native threads)],
 937         description => 'AIO type to use.',
 938         optional => 1,
 939     },
 940     discard => {
 941         type => 'string',
 942         enum => [qw(ignore on)],
 943         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 944         optional => 1,
 945     },
 946     detect_zeroes => {
 947         type => 'boolean',
 948         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 949         optional => 1,
 950     },
 951     serial => {
 952         type => 'string',
 953         format => 'urlencoded',
 954         format_description => 'serial',
 955         maxLength => 20*3, # *3 since it's %xx url enoded
 956         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 957         optional => 1,
 958     },
 959     shared => {
 960         type => 'boolean',
 961         description => 'Mark this locally-managed volume as available on all nodes',
 962         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 963         optional => 1,
 964         default => 0,
 965     }
 966 );
 967
 968 my %iothread_fmt = ( iothread => {
 969         type => 'boolean',
 970         description => "Whether to use iothreads for this drive",
 971         optional => 1,
 972 });
 973
 974 my %model_fmt = (
 975     model => {
 976         type => 'string',
 977         format => 'urlencoded',
 978         format_description => 'model',
 979         maxLength => 40*3, # *3 since it's %xx url enoded
 980         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 981         optional => 1,
 982     },
 983 );
 984
 985 my %queues_fmt = (
 986     queues => {
 987         type => 'integer',
 988         description => "Number of queues.",
 989         minimum => 2,
 990         optional => 1
 991     }
 992 );
 993
 994 my %scsiblock_fmt = (
 995     scsiblock => {
 996         type => 'boolean',
 997         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 998         optional => 1,
 999         default => 0,
1000     },
1001 );
1002
1003 my $add_throttle_desc = sub {
1004     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1005     my $d = {
1006         type => $type,
1007         format_description => $unit,
1008         description => "Maximum $what in $longunit.",
1009         optional => 1,
1010     };
1011     $d->{minimum} = $minimum if defined($minimum);
1012     $drivedesc_base{$key} = $d;
1013 };
1014 # throughput: (leaky bucket)
1015 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1016 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1017 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1018 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1019 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1020 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1021 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1022 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1023 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1024
1025 # pools: (pool of IO before throttling starts taking effect)
1026 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1027 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1028 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1029 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1030 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1031 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1032
1033 # burst lengths
1034 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1035 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1036 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1037 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1038 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1039 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1040
1041 # legacy support
1042 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1043 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1044 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1045 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1046
1047 my $ide_fmt = {
1048     %drivedesc_base,
1049     %model_fmt,
1050 };
1051 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1052
1053 my $idedesc = {
1054     optional => 1,
1055     type => 'string', format => $ide_fmt,
1056     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1057 };
1058 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1059
1060 my $scsi_fmt = {
1061     %drivedesc_base,
1062     %iothread_fmt,
1063     %queues_fmt,
1064     %scsiblock_fmt,
1065 };
1066 my $scsidesc = {
1067     optional => 1,
1068     type => 'string', format => $scsi_fmt,
1069     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1070 };
1071 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1072
1073 my $sata_fmt = {
1074     %drivedesc_base,
1075 };
1076 my $satadesc = {
1077     optional => 1,
1078     type => 'string', format => $sata_fmt,
1079     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1080 };
1081 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1082
1083 my $virtio_fmt = {
1084     %drivedesc_base,
1085     %iothread_fmt,
1086 };
1087 my $virtiodesc = {
1088     optional => 1,
1089     type => 'string', format => $virtio_fmt,
1090     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1091 };
1092 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1093
1094 my $alldrive_fmt = {
1095     %drivedesc_base,
1096     %iothread_fmt,
1097     %model_fmt,
1098     %queues_fmt,
1099     %scsiblock_fmt,
1100 };
1101
1102 my $efidisk_fmt = {
1103     volume => { alias => 'file' },
1104     file => {
1105         type => 'string',
1106         format => 'pve-volume-id-or-qm-path',
1107         default_key => 1,
1108         format_description => 'volume',
1109         description => "The drive's backing volume.",
1110     },
1111     format => get_standard_option('pve-qm-image-format'),
1112     size => {
1113         type => 'string',
1114         format => 'disk-size',
1115         format_description => 'DiskSize',
1116         description => "Disk size. This is purely informational and has no effect.",
1117         optional => 1,
1118     },
1119 };
1120
1121 my $efidisk_desc = {
1122     optional => 1,
1123     type => 'string', format => $efidisk_fmt,
1124     description => "Configure a Disk for storing EFI vars",
1125 };
1126
1127 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1128
1129 my $usb_fmt = {
1130     host => {
1131         default_key => 1,
1132         type => 'string', format => 'pve-qm-usb-device',
1133         format_description => 'HOSTUSBDEVICE|spice',
1134         description => <<EODESCR,
1135 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1136
1137  'bus-port(.port)*' (decimal numbers) or
1138  'vendor_id:product_id' (hexadeciaml numbers) or
1139  'spice'
1140
1141 You can use the 'lsusb -t' command to list existing usb devices.
1142
1143 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1144
1145 The value 'spice' can be used to add a usb redirection devices for spice.
1146 EODESCR
1147     },
1148     usb3 => {
1149         optional => 1,
1150         type => 'boolean',
1151         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1152         default => 0,
1153     },
1154 };
1155
1156 my $usbdesc = {
1157     optional => 1,
1158     type => 'string', format => $usb_fmt,
1159     description => "Configure an USB device (n is 0 to 4).",
1160 };
1161 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1162
1163 # NOTE: the match-groups of this regex are used in parse_hostpci
1164 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1165 my $hostpci_fmt = {
1166     host => {
1167         default_key => 1,
1168         type => 'string',
1169         pattern => qr/$PCIRE(;$PCIRE)*/,
1170         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1171         description => <<EODESCR,
1172 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1173 of PCI virtual functions of the host. HOSTPCIID syntax is:
1174
1175 'bus:dev.func' (hexadecimal numbers)
1176
1177 You can us the 'lspci' command to list existing PCI devices.
1178 EODESCR
1179     },
1180     rombar => {
1181         type => 'boolean',
1182         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1183         optional => 1,
1184         default => 1,
1185     },
1186     romfile => {
1187         type => 'string',
1188         pattern => '[^,;]+',
1189         format_description => 'string',
1190         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1191         optional => 1,
1192     },
1193     pcie => {
1194         type => 'boolean',
1195         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1196         optional => 1,
1197         default => 0,
1198     },
1199     'x-vga' => {
1200         type => 'boolean',
1201         description =>  "Enable vfio-vga device support.",
1202         optional => 1,
1203         default => 0,
1204     },
1205 };
1206 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1207
1208 my $hostpcidesc = {
1209         optional => 1,
1210         type => 'string', format => 'pve-qm-hostpci',
1211         description => "Map host PCI devices into guest.",
1212         verbose_description =>  <<EODESCR,
1213 Map host PCI devices into guest.
1214
1215 NOTE: This option allows direct access to host hardware. So it is no longer
1216 possible to migrate such machines - use with special care.
1217
1218 CAUTION: Experimental! User reported problems with this option.
1219 EODESCR
1220 };
1221 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1222
1223 my $serialdesc = {
1224         optional => 1,
1225         type => 'string',
1226         pattern => '(/dev/.+|socket)',
1227         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1228         verbose_description =>  <<EODESCR,
1229 Create a serial device inside the VM (n is 0 to 3), and pass through a
1230 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1231 host side (use 'qm terminal' to open a terminal connection).
1232
1233 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1234
1235 CAUTION: Experimental! User reported problems with this option.
1236 EODESCR
1237 };
1238
1239 my $paralleldesc= {
1240         optional => 1,
1241         type => 'string',
1242         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1243         description =>  "Map host parallel devices (n is 0 to 2).",
1244         verbose_description =>  <<EODESCR,
1245 Map host parallel devices (n is 0 to 2).
1246
1247 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1248
1249 CAUTION: Experimental! User reported problems with this option.
1250 EODESCR
1251 };
1252
1253 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1254     $confdesc->{"parallel$i"} = $paralleldesc;
1255 }
1256
1257 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1258     $confdesc->{"serial$i"} = $serialdesc;
1259 }
1260
1261 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1262     $confdesc->{"hostpci$i"} = $hostpcidesc;
1263 }
1264
1265 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1266     $drivename_hash->{"ide$i"} = 1;
1267     $confdesc->{"ide$i"} = $idedesc;
1268 }
1269
1270 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1271     $drivename_hash->{"sata$i"} = 1;
1272     $confdesc->{"sata$i"} = $satadesc;
1273 }
1274
1275 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1276     $drivename_hash->{"scsi$i"} = 1;
1277     $confdesc->{"scsi$i"} = $scsidesc ;
1278 }
1279
1280 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1281     $drivename_hash->{"virtio$i"} = 1;
1282     $confdesc->{"virtio$i"} = $virtiodesc;
1283 }
1284
1285 $drivename_hash->{efidisk0} = 1;
1286 $confdesc->{efidisk0} = $efidisk_desc;
1287
1288 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1289     $confdesc->{"usb$i"} = $usbdesc;
1290 }
1291
1292 my $unuseddesc = {
1293     optional => 1,
1294     type => 'string', format => 'pve-volume-id',
1295     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1296 };
1297
1298 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1299     $confdesc->{"unused$i"} = $unuseddesc;
1300 }
1301
1302 my $kvm_api_version = 0;
1303
1304 sub kvm_version {
1305
1306     return $kvm_api_version if $kvm_api_version;
1307
1308     my $fh = IO::File->new("</dev/kvm") ||
1309         return 0;
1310
1311     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1312         $kvm_api_version = $v;
1313     }
1314
1315     $fh->close();
1316
1317     return  $kvm_api_version;
1318 }
1319
1320 my $kvm_user_version;
1321
1322 sub kvm_user_version {
1323
1324     return $kvm_user_version if $kvm_user_version;
1325
1326     $kvm_user_version = 'unknown';
1327
1328     my $code = sub {
1329         my $line = shift;
1330         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1331             $kvm_user_version = $2;
1332         }
1333     };
1334
1335     eval { run_command("kvm -version", outfunc => $code); };
1336     warn $@ if $@;
1337
1338     return $kvm_user_version;
1339
1340 }
1341
1342 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1343
1344 sub valid_drive_names {
1345     # order is important - used to autoselect boot disk
1346     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1347             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1348             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1349             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1350             'efidisk0');
1351 }
1352
1353 sub is_valid_drivename {
1354     my $dev = shift;
1355
1356     return defined($drivename_hash->{$dev});
1357 }
1358
1359 sub option_exists {
1360     my $key = shift;
1361     return defined($confdesc->{$key});
1362 }
1363
1364 sub nic_models {
1365     return $nic_model_list;
1366 }
1367
1368 sub os_list_description {
1369
1370     return {
1371         other => 'Other',
1372         wxp => 'Windows XP',
1373         w2k => 'Windows 2000',
1374         w2k3 =>, 'Windows 2003',
1375         w2k8 => 'Windows 2008',
1376         wvista => 'Windows Vista',
1377         win7 => 'Windows 7',
1378         win8 => 'Windows 8/2012',
1379         win10 => 'Windows 10/2016',
1380         l24 => 'Linux 2.4',
1381         l26 => 'Linux 2.6',
1382     };
1383 }
1384
1385 my $cdrom_path;
1386
1387 sub get_cdrom_path {
1388
1389     return  $cdrom_path if $cdrom_path;
1390
1391     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1392     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1393     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1394 }
1395
1396 sub get_iso_path {
1397     my ($storecfg, $vmid, $cdrom) = @_;
1398
1399     if ($cdrom eq 'cdrom') {
1400         return get_cdrom_path();
1401     } elsif ($cdrom eq 'none') {
1402         return '';
1403     } elsif ($cdrom =~ m|^/|) {
1404         return $cdrom;
1405     } else {
1406         return PVE::Storage::path($storecfg, $cdrom);
1407     }
1408 }
1409
1410 # try to convert old style file names to volume IDs
1411 sub filename_to_volume_id {
1412     my ($vmid, $file, $media) = @_;
1413
1414      if (!($file eq 'none' || $file eq 'cdrom' ||
1415           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1416
1417         return undef if $file =~ m|/|;
1418
1419         if ($media && $media eq 'cdrom') {
1420             $file = "local:iso/$file";
1421         } else {
1422             $file = "local:$vmid/$file";
1423         }
1424     }
1425
1426     return $file;
1427 }
1428
1429 sub verify_media_type {
1430     my ($opt, $vtype, $media) = @_;
1431
1432     return if !$media;
1433
1434     my $etype;
1435     if ($media eq 'disk') {
1436         $etype = 'images';
1437     } elsif ($media eq 'cdrom') {
1438         $etype = 'iso';
1439     } else {
1440         die "internal error";
1441     }
1442
1443     return if ($vtype eq $etype);
1444
1445     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1446 }
1447
1448 sub cleanup_drive_path {
1449     my ($opt, $storecfg, $drive) = @_;
1450
1451     # try to convert filesystem paths to volume IDs
1452
1453     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1454         ($drive->{file} !~ m|^/dev/.+|) &&
1455         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1456         ($drive->{file} !~ m/^\d+$/)) {
1457         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1458         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1459         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1460         verify_media_type($opt, $vtype, $drive->{media});
1461         $drive->{file} = $volid;
1462     }
1463
1464     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1465 }
1466
1467 sub parse_hotplug_features {
1468     my ($data) = @_;
1469
1470     my $res = {};
1471
1472     return $res if $data eq '0';
1473
1474     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1475
1476     foreach my $feature (PVE::Tools::split_list($data)) {
1477         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1478             $res->{$1} = 1;
1479         } else {
1480             die "invalid hotplug feature '$feature'\n";
1481         }
1482     }
1483     return $res;
1484 }
1485
1486 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1487 sub pve_verify_hotplug_features {
1488     my ($value, $noerr) = @_;
1489
1490     return $value if parse_hotplug_features($value);
1491
1492     return undef if $noerr;
1493
1494     die "unable to parse hotplug option\n";
1495 }
1496
1497 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1498 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1499 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1500 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1501 #        [,iothread=on][,serial=serial][,model=model]
1502
1503 sub parse_drive {
1504     my ($key, $data) = @_;
1505
1506     my ($interface, $index);
1507
1508     if ($key =~ m/^([^\d]+)(\d+)$/) {
1509         $interface = $1;
1510         $index = $2;
1511     } else {
1512         return undef;
1513     }
1514
1515     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1516                                      : $confdesc->{$key}->{format};
1517     if (!$desc) {
1518         warn "invalid drive key: $key\n";
1519         return undef;
1520     }
1521     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1522     return undef if !$res;
1523     $res->{interface} = $interface;
1524     $res->{index} = $index;
1525
1526     my $error = 0;
1527     foreach my $opt (qw(bps bps_rd bps_wr)) {
1528         if (my $bps = defined(delete $res->{$opt})) {
1529             if (defined($res->{"m$opt"})) {
1530                 warn "both $opt and m$opt specified\n";
1531                 ++$error;
1532                 next;
1533             }
1534             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1535         }
1536     }
1537
1538     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1539     for my $requirement (
1540         [mbps_max => 'mbps'],
1541         [mbps_rd_max => 'mbps_rd'],
1542         [mbps_wr_max => 'mbps_wr'],
1543         [miops_max => 'miops'],
1544         [miops_rd_max => 'miops_rd'],
1545         [miops_wr_max => 'miops_wr'],
1546         [bps_max_length => 'mbps_max'],
1547         [bps_rd_max_length => 'mbps_rd_max'],
1548         [bps_wr_max_length => 'mbps_wr_max'],
1549         [iops_max_length => 'iops_max'],
1550         [iops_rd_max_length => 'iops_rd_max'],
1551         [iops_wr_max_length => 'iops_wr_max']) {
1552         my ($option, $requires) = @$requirement;
1553         if ($res->{$option} && !$res->{$requires}) {
1554             warn "$option requires $requires\n";
1555             ++$error;
1556         }
1557     }
1558
1559     return undef if $error;
1560
1561     return undef if $res->{mbps_rd} && $res->{mbps};
1562     return undef if $res->{mbps_wr} && $res->{mbps};
1563     return undef if $res->{iops_rd} && $res->{iops};
1564     return undef if $res->{iops_wr} && $res->{iops};
1565
1566     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1567         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1568         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1569         return undef if $res->{interface} eq 'virtio';
1570     }
1571
1572     if (my $size = $res->{size}) {
1573         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1574     }
1575
1576     return $res;
1577 }
1578
1579 sub print_drive {
1580     my ($vmid, $drive) = @_;
1581     my $data = { %$drive };
1582     delete $data->{$_} for qw(index interface);
1583     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1584 }
1585
1586 sub scsi_inquiry {
1587     my($fh, $noerr) = @_;
1588
1589     my $SG_IO = 0x2285;
1590     my $SG_GET_VERSION_NUM = 0x2282;
1591
1592     my $versionbuf = "\x00" x 8;
1593     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1594     if (!$ret) {
1595         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1596         return undef;
1597     }
1598     my $version = unpack("I", $versionbuf);
1599     if ($version < 30000) {
1600         die "scsi generic interface too old\n"  if !$noerr;
1601         return undef;
1602     }
1603
1604     my $buf = "\x00" x 36;
1605     my $sensebuf = "\x00" x 8;
1606     my $cmd = pack("C x3 C x1", 0x12, 36);
1607
1608     # see /usr/include/scsi/sg.h
1609     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1610
1611     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1612                       length($sensebuf), 0, length($buf), $buf,
1613                       $cmd, $sensebuf, 6000);
1614
1615     $ret = ioctl($fh, $SG_IO, $packet);
1616     if (!$ret) {
1617         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1618         return undef;
1619     }
1620
1621     my @res = unpack($sg_io_hdr_t, $packet);
1622     if ($res[17] || $res[18]) {
1623         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1624         return undef;
1625     }
1626
1627     my $res = {};
1628     (my $byte0, my $byte1, $res->{vendor},
1629      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1630
1631     $res->{removable} = $byte1 & 128 ? 1 : 0;
1632     $res->{type} = $byte0 & 31;
1633
1634     return $res;
1635 }
1636
1637 sub path_is_scsi {
1638     my ($path) = @_;
1639
1640     my $fh = IO::File->new("+<$path") || return undef;
1641     my $res = scsi_inquiry($fh, 1);
1642     close($fh);
1643
1644     return $res;
1645 }
1646
1647 sub machine_type_is_q35 {
1648     my ($conf) = @_;
1649
1650     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1651 }
1652
1653 sub print_tabletdevice_full {
1654     my ($conf) = @_;
1655
1656     my $q35 = machine_type_is_q35($conf);
1657
1658     # we use uhci for old VMs because tablet driver was buggy in older qemu
1659     my $usbbus = $q35 ? "ehci" : "uhci";
1660
1661     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1662 }
1663
1664 sub print_drivedevice_full {
1665     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1666
1667     my $device = '';
1668     my $maxdev = 0;
1669
1670     if ($drive->{interface} eq 'virtio') {
1671         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1672         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1673         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1674     } elsif ($drive->{interface} eq 'scsi') {
1675
1676         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1677         my $unit = $drive->{index} % $maxdev;
1678         my $devicetype = 'hd';
1679         my $path = '';
1680         if (drive_is_cdrom($drive)) {
1681             $devicetype = 'cd';
1682         } else {
1683             if ($drive->{file} =~ m|^/|) {
1684                 $path = $drive->{file};
1685                 if (my $info = path_is_scsi($path)) {
1686                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1687                         $devicetype = 'block';
1688                     } elsif ($info->{type} == 1) { # tape
1689                         $devicetype = 'generic';
1690                     }
1691                 }
1692             } else {
1693                  $path = PVE::Storage::path($storecfg, $drive->{file});
1694             }
1695
1696             if($path =~ m/^iscsi\:\/\//){
1697                 $devicetype = 'generic';
1698             }
1699         }
1700
1701         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1702             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1703         } else {
1704             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1705         }
1706
1707     } elsif ($drive->{interface} eq 'ide'){
1708         $maxdev = 2;
1709         my $controller = int($drive->{index} / $maxdev);
1710         my $unit = $drive->{index} % $maxdev;
1711         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1712
1713         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1714         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1715             $model = URI::Escape::uri_unescape($model);
1716             $device .= ",model=$model";
1717         }
1718     } elsif ($drive->{interface} eq 'sata'){
1719         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1720         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1721         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1722     } elsif ($drive->{interface} eq 'usb') {
1723         die "implement me";
1724         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1725     } else {
1726         die "unsupported interface type";
1727     }
1728
1729     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1730
1731     if (my $serial = $drive->{serial}) {
1732         $serial = URI::Escape::uri_unescape($serial);
1733         $device .= ",serial=$serial";
1734     }
1735
1736
1737     return $device;
1738 }
1739
1740 sub get_initiator_name {
1741     my $initiator;
1742
1743     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1744     while (defined(my $line = <$fh>)) {
1745         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1746         $initiator = $1;
1747         last;
1748     }
1749     $fh->close();
1750
1751     return $initiator;
1752 }
1753
1754 sub print_drive_full {
1755     my ($storecfg, $vmid, $drive) = @_;
1756
1757     my $path;
1758     my $volid = $drive->{file};
1759     my $format;
1760
1761     if (drive_is_cdrom($drive)) {
1762         $path = get_iso_path($storecfg, $vmid, $volid);
1763     } else {
1764         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1765         if ($storeid) {
1766             $path = PVE::Storage::path($storecfg, $volid);
1767             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1768             $format = qemu_img_format($scfg, $volname);
1769         } else {
1770             $path = $volid;
1771             $format = "raw";
1772         }
1773    }
1774
1775     my $opts = '';
1776     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1777     foreach my $o (@qemu_drive_options) {
1778         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1779     }
1780
1781     # snapshot only accepts on|off
1782     if (defined($drive->{snapshot})) {
1783         my $v = $drive->{snapshot} ? 'on' : 'off';
1784         $opts .= ",snapshot=$v";
1785     }
1786
1787     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1788         my ($dir, $qmpname) = @$type;
1789         if (my $v = $drive->{"mbps$dir"}) {
1790             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1791         }
1792         if (my $v = $drive->{"mbps${dir}_max"}) {
1793             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1794         }
1795         if (my $v = $drive->{"bps${dir}_max_length"}) {
1796             $opts .= ",throttling.bps$qmpname-max-length=$v";
1797         }
1798         if (my $v = $drive->{"iops${dir}"}) {
1799             $opts .= ",throttling.iops$qmpname=$v";
1800         }
1801         if (my $v = $drive->{"iops${dir}_max"}) {
1802             $opts .= ",throttling.iops$qmpname-max=$v";
1803         }
1804         if (my $v = $drive->{"iops${dir}_max_length"}) {
1805             $opts .= ",throttling.iops$qmpname-max-length=$v";
1806         }
1807     }
1808
1809     $opts .= ",format=$format" if $format && !$drive->{format};
1810
1811     my $cache_direct = 0;
1812
1813     if (my $cache = $drive->{cache}) {
1814         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1815     } elsif (!drive_is_cdrom($drive)) {
1816         $opts .= ",cache=none";
1817         $cache_direct = 1;
1818     }
1819
1820     # aio native works only with O_DIRECT
1821     if (!$drive->{aio}) {
1822         if($cache_direct) {
1823             $opts .= ",aio=native";
1824         } else {
1825             $opts .= ",aio=threads";
1826         }
1827     }
1828
1829     if (!drive_is_cdrom($drive)) {
1830         my $detectzeroes;
1831         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1832             $detectzeroes = 'off';
1833         } elsif ($drive->{discard}) {
1834             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1835         } else {
1836             # This used to be our default with discard not being specified:
1837             $detectzeroes = 'on';
1838         }
1839         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1840     }
1841
1842     my $pathinfo = $path ? "file=$path," : '';
1843
1844     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1845 }
1846
1847 sub print_netdevice_full {
1848     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1849
1850     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1851
1852     my $device = $net->{model};
1853     if ($net->{model} eq 'virtio') {
1854          $device = 'virtio-net-pci';
1855      };
1856
1857     my $pciaddr = print_pci_addr("$netid", $bridges);
1858     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1859     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1860         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1861         my $vectors = $net->{queues} * 2 + 2;
1862         $tmpstr .= ",vectors=$vectors,mq=on";
1863     }
1864     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1865
1866     if ($use_old_bios_files) {
1867         my $romfile;
1868         if ($device eq 'virtio-net-pci') {
1869             $romfile = 'pxe-virtio.rom';
1870         } elsif ($device eq 'e1000') {
1871             $romfile = 'pxe-e1000.rom';
1872         } elsif ($device eq 'ne2k') {
1873             $romfile = 'pxe-ne2k_pci.rom';
1874         } elsif ($device eq 'pcnet') {
1875             $romfile = 'pxe-pcnet.rom';
1876         } elsif ($device eq 'rtl8139') {
1877             $romfile = 'pxe-rtl8139.rom';
1878         }
1879         $tmpstr .= ",romfile=$romfile" if $romfile;
1880     }
1881
1882     return $tmpstr;
1883 }
1884
1885 sub print_netdev_full {
1886     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1887
1888     my $i = '';
1889     if ($netid =~ m/^net(\d+)$/) {
1890         $i = int($1);
1891     }
1892
1893     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1894
1895     my $ifname = "tap${vmid}i$i";
1896
1897     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1898     die "interface name '$ifname' is too long (max 15 character)\n"
1899         if length($ifname) >= 16;
1900
1901     my $vhostparam = '';
1902     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1903
1904     my $vmname = $conf->{name} || "vm$vmid";
1905
1906     my $netdev = "";
1907     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1908
1909     if ($net->{bridge}) {
1910         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1911     } else {
1912         $netdev = "type=user,id=$netid,hostname=$vmname";
1913     }
1914
1915     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1916
1917     return $netdev;
1918 }
1919
1920
1921 sub print_cpu_device {
1922     my ($conf, $id) = @_;
1923
1924     my $kvm = $conf->{kvm} // 1;
1925     my $cpu = $kvm ? "kvm64" : "qemu64";
1926     if (my $cputype = $conf->{cpu}) {
1927         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1928             or die "Cannot parse cpu description: $cputype\n";
1929         $cpu = $cpuconf->{cputype};
1930     }
1931
1932     my $cores = $conf->{cores} || 1;
1933
1934     my $current_core = ($id - 1) % $cores;
1935     my $current_socket = int(($id - 1 - $current_core)/$cores);
1936
1937     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1938 }
1939
1940 sub drive_is_cloudinit {
1941     my ($drive) = @_;
1942     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1943 }
1944
1945 sub drive_is_cdrom {
1946     my ($drive, $exclude_cloudinit) = @_;
1947
1948     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1949
1950     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1951
1952 }
1953
1954 sub parse_number_sets {
1955     my ($set) = @_;
1956     my $res = [];
1957     foreach my $part (split(/;/, $set)) {
1958         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1959             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1960             push @$res, [ $1, $2 ];
1961         } else {
1962             die "invalid range: $part\n";
1963         }
1964     }
1965     return $res;
1966 }
1967
1968 sub parse_numa {
1969     my ($data) = @_;
1970
1971     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1972     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1973     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1974     return $res;
1975 }
1976
1977 sub parse_hostpci {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1983
1984     my @idlist = split(/;/, $res->{host});
1985     delete $res->{host};
1986     foreach my $id (@idlist) {
1987         if ($id =~ /^$PCIRE$/) {
1988             if (defined($2)) {
1989                 push @{$res->{pciid}}, { id => $1, function => $2 };
1990             } else {
1991                 my $pcidevices = lspci($1);
1992                 $res->{pciid} = $pcidevices->{$1};
1993             }
1994         } else {
1995             # should have been caught by parse_property_string already
1996             die "failed to parse PCI id: $id\n";
1997         }
1998     }
1999     return $res;
2000 }
2001
2002 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2003 sub parse_net {
2004     my ($data) = @_;
2005
2006     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2007     if ($@) {
2008         warn $@;
2009         return undef;
2010     }
2011     if (!defined($res->{macaddr})) {
2012         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2013         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2014     }
2015     return $res;
2016 }
2017
2018 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2019 sub parse_ipconfig {
2020     my ($data) = @_;
2021
2022     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2023     if ($@) {
2024         warn $@;
2025         return undef;
2026     }
2027
2028     if ($res->{gw} && !$res->{ip}) {
2029         warn 'gateway specified without specifying an IP address';
2030         return undef;
2031     }
2032     if ($res->{gw6} && !$res->{ip6}) {
2033         warn 'IPv6 gateway specified without specifying an IPv6 address';
2034         return undef;
2035     }
2036     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2037         warn 'gateway specified together with DHCP';
2038         return undef;
2039     }
2040     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2041         # gw6 + auto/dhcp
2042         warn "IPv6 gateway specified together with $res->{ip6} address";
2043         return undef;
2044     }
2045
2046     if (!$res->{ip} && !$res->{ip6}) {
2047         return { ip => 'dhcp', ip6 => 'dhcp' };
2048     }
2049
2050     return $res;
2051 }
2052
2053 sub print_net {
2054     my $net = shift;
2055
2056     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2057 }
2058
2059 sub add_random_macs {
2060     my ($settings) = @_;
2061
2062     foreach my $opt (keys %$settings) {
2063         next if $opt !~ m/^net(\d+)$/;
2064         my $net = parse_net($settings->{$opt});
2065         next if !$net;
2066         $settings->{$opt} = print_net($net);
2067     }
2068 }
2069
2070 sub vm_is_volid_owner {
2071     my ($storecfg, $vmid, $volid) = @_;
2072
2073     if ($volid !~  m|^/|) {
2074         my ($path, $owner);
2075         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2076         if ($owner && ($owner == $vmid)) {
2077             return 1;
2078         }
2079     }
2080
2081     return undef;
2082 }
2083
2084 sub split_flagged_list {
2085     my $text = shift || '';
2086     $text =~ s/[,;]/ /g;
2087     $text =~ s/^\s+//;
2088     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2089 }
2090
2091 sub join_flagged_list {
2092     my ($how, $lst) = @_;
2093     join $how, map { $lst->{$_} . $_ } keys %$lst;
2094 }
2095
2096 sub vmconfig_delete_pending_option {
2097     my ($conf, $key, $force) = @_;
2098
2099     delete $conf->{pending}->{$key};
2100     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2101     $pending_delete_hash->{$key} = $force ? '!' : '';
2102     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2103 }
2104
2105 sub vmconfig_undelete_pending_option {
2106     my ($conf, $key) = @_;
2107
2108     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2109     delete $pending_delete_hash->{$key};
2110
2111     if (%$pending_delete_hash) {
2112         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2113     } else {
2114         delete $conf->{pending}->{delete};
2115     }
2116 }
2117
2118 sub vmconfig_register_unused_drive {
2119     my ($storecfg, $vmid, $conf, $drive) = @_;
2120
2121     if (drive_is_cloudinit($drive)) {
2122         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2123         warn $@ if $@;
2124     } elsif (!drive_is_cdrom($drive)) {
2125         my $volid = $drive->{file};
2126         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2127             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2128         }
2129     }
2130 }
2131
2132 sub vmconfig_cleanup_pending {
2133     my ($conf) = @_;
2134
2135     # remove pending changes when nothing changed
2136     my $changes;
2137     foreach my $opt (keys %{$conf->{pending}}) {
2138         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2139             $changes = 1;
2140             delete $conf->{pending}->{$opt};
2141         }
2142     }
2143
2144     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2145     my $pending_delete_hash = {};
2146     while (my ($opt, $force) = each %$current_delete_hash) {
2147         if (defined($conf->{$opt})) {
2148             $pending_delete_hash->{$opt} = $force;
2149         } else {
2150             $changes = 1;
2151         }
2152     }
2153
2154     if (%$pending_delete_hash) {
2155         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2156     } else {
2157         delete $conf->{pending}->{delete};
2158     }
2159
2160     return $changes;
2161 }
2162
2163 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2164 my $smbios1_fmt = {
2165     uuid => {
2166         type => 'string',
2167         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2168         format_description => 'UUID',
2169         description => "Set SMBIOS1 UUID.",
2170         optional => 1,
2171     },
2172     version => {
2173         type => 'string',
2174         pattern => '\S+',
2175         format_description => 'string',
2176         description => "Set SMBIOS1 version.",
2177         optional => 1,
2178     },
2179     serial => {
2180         type => 'string',
2181         pattern => '\S+',
2182         format_description => 'string',
2183         description => "Set SMBIOS1 serial number.",
2184         optional => 1,
2185     },
2186     manufacturer => {
2187         type => 'string',
2188         pattern => '\S+',
2189         format_description => 'string',
2190         description => "Set SMBIOS1 manufacturer.",
2191         optional => 1,
2192     },
2193     product => {
2194         type => 'string',
2195         pattern => '\S+',
2196         format_description => 'string',
2197         description => "Set SMBIOS1 product ID.",
2198         optional => 1,
2199     },
2200     sku => {
2201         type => 'string',
2202         pattern => '\S+',
2203         format_description => 'string',
2204         description => "Set SMBIOS1 SKU string.",
2205         optional => 1,
2206     },
2207     family => {
2208         type => 'string',
2209         pattern => '\S+',
2210         format_description => 'string',
2211         description => "Set SMBIOS1 family string.",
2212         optional => 1,
2213     },
2214 };
2215
2216 sub parse_smbios1 {
2217     my ($data) = @_;
2218
2219     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2220     warn $@ if $@;
2221     return $res;
2222 }
2223
2224 sub print_smbios1 {
2225     my ($smbios1) = @_;
2226     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2227 }
2228
2229 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2230
2231 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2232 sub verify_bootdisk {
2233     my ($value, $noerr) = @_;
2234
2235     return $value if is_valid_drivename($value);
2236
2237     return undef if $noerr;
2238
2239     die "invalid boot disk '$value'\n";
2240 }
2241
2242 sub parse_watchdog {
2243     my ($value) = @_;
2244
2245     return undef if !$value;
2246
2247     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2248     warn $@ if $@;
2249     return $res;
2250 }
2251
2252 sub parse_guest_agent {
2253     my ($value) = @_;
2254
2255     return {} if !defined($value->{agent});
2256
2257     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2258     warn $@ if $@;
2259
2260     # if the agent is disabled ignore the other potentially set properties
2261     return {} if !$res->{enabled};
2262     return $res;
2263 }
2264
2265 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2266 sub verify_usb_device {
2267     my ($value, $noerr) = @_;
2268
2269     return $value if parse_usb_device($value);
2270
2271     return undef if $noerr;
2272
2273     die "unable to parse usb device\n";
2274 }
2275
2276 # add JSON properties for create and set function
2277 sub json_config_properties {
2278     my $prop = shift;
2279
2280     foreach my $opt (keys %$confdesc) {
2281         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2282         $prop->{$opt} = $confdesc->{$opt};
2283     }
2284
2285     return $prop;
2286 }
2287
2288 # return copy of $confdesc_cloudinit to generate documentation
2289 sub cloudinit_config_properties {
2290
2291     return dclone($confdesc_cloudinit);
2292 }
2293
2294 sub check_type {
2295     my ($key, $value) = @_;
2296
2297     die "unknown setting '$key'\n" if !$confdesc->{$key};
2298
2299     my $type = $confdesc->{$key}->{type};
2300
2301     if (!defined($value)) {
2302         die "got undefined value\n";
2303     }
2304
2305     if ($value =~ m/[\n\r]/) {
2306         die "property contains a line feed\n";
2307     }
2308
2309     if ($type eq 'boolean') {
2310         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2311         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2312         die "type check ('boolean') failed - got '$value'\n";
2313     } elsif ($type eq 'integer') {
2314         return int($1) if $value =~ m/^(\d+)$/;
2315         die "type check ('integer') failed - got '$value'\n";
2316     } elsif ($type eq 'number') {
2317         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2318         die "type check ('number') failed - got '$value'\n";
2319     } elsif ($type eq 'string') {
2320         if (my $fmt = $confdesc->{$key}->{format}) {
2321             PVE::JSONSchema::check_format($fmt, $value);
2322             return $value;
2323         }
2324         $value =~ s/^\"(.*)\"$/$1/;
2325         return $value;
2326     } else {
2327         die "internal error"
2328     }
2329 }
2330
2331 sub check_iommu_support{
2332     #fixme : need to check IOMMU support
2333     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2334
2335     my $iommu=1;
2336     return $iommu;
2337
2338 }
2339
2340 sub touch_config {
2341     my ($vmid) = @_;
2342
2343     my $conf = PVE::QemuConfig->config_file($vmid);
2344     utime undef, undef, $conf;
2345 }
2346
2347 sub destroy_vm {
2348     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2349
2350     my $conffile = PVE::QemuConfig->config_file($vmid);
2351
2352     my $conf = PVE::QemuConfig->load_config($vmid);
2353
2354     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2355
2356     if ($conf->{template}) {
2357         # check if any base image is still used by a linked clone
2358         foreach_drive($conf, sub {
2359                 my ($ds, $drive) = @_;
2360
2361                 return if drive_is_cdrom($drive);
2362
2363                 my $volid = $drive->{file};
2364
2365                 return if !$volid || $volid =~ m|^/|;
2366
2367                 die "base volume '$volid' is still in use by linked cloned\n"
2368                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2369
2370         });
2371     }
2372
2373     # only remove disks owned by this VM
2374     foreach_drive($conf, sub {
2375         my ($ds, $drive) = @_;
2376
2377         return if drive_is_cdrom($drive, 1);
2378
2379         my $volid = $drive->{file};
2380
2381         return if !$volid || $volid =~ m|^/|;
2382
2383         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2384         return if !$path || !$owner || ($owner != $vmid);
2385
2386         eval {
2387             PVE::Storage::vdisk_free($storecfg, $volid);
2388         };
2389         warn "Could not remove disk '$volid', check manually: $@" if $@;
2390
2391     });
2392
2393     if ($keep_empty_config) {
2394         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2395     } else {
2396         unlink $conffile;
2397     }
2398
2399     # also remove unused disk
2400     eval {
2401         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2402
2403         eval {
2404             PVE::Storage::foreach_volid($dl, sub {
2405                 my ($volid, $sid, $volname, $d) = @_;
2406                 PVE::Storage::vdisk_free($storecfg, $volid);
2407             });
2408         };
2409         warn $@ if $@;
2410
2411     };
2412     warn $@ if $@;
2413 }
2414
2415 sub parse_vm_config {
2416     my ($filename, $raw) = @_;
2417
2418     return undef if !defined($raw);
2419
2420     my $res = {
2421         digest => Digest::SHA::sha1_hex($raw),
2422         snapshots => {},
2423         pending => {},
2424     };
2425
2426     $filename =~ m|/qemu-server/(\d+)\.conf$|
2427         || die "got strange filename '$filename'";
2428
2429     my $vmid = $1;
2430
2431     my $conf = $res;
2432     my $descr;
2433     my $section = '';
2434
2435     my @lines = split(/\n/, $raw);
2436     foreach my $line (@lines) {
2437         next if $line =~ m/^\s*$/;
2438
2439         if ($line =~ m/^\[PENDING\]\s*$/i) {
2440             $section = 'pending';
2441             if (defined($descr)) {
2442                 $descr =~ s/\s+$//;
2443                 $conf->{description} = $descr;
2444             }
2445             $descr = undef;
2446             $conf = $res->{$section} = {};
2447             next;
2448
2449         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2450             $section = $1;
2451             if (defined($descr)) {
2452                 $descr =~ s/\s+$//;
2453                 $conf->{description} = $descr;
2454             }
2455             $descr = undef;
2456             $conf = $res->{snapshots}->{$section} = {};
2457             next;
2458         }
2459
2460         if ($line =~ m/^\#(.*)\s*$/) {
2461             $descr = '' if !defined($descr);
2462             $descr .= PVE::Tools::decode_text($1) . "\n";
2463             next;
2464         }
2465
2466         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2467             $descr = '' if !defined($descr);
2468             $descr .= PVE::Tools::decode_text($2);
2469         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2470             $conf->{snapstate} = $1;
2471         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2472             my $key = $1;
2473             my $value = $2;
2474             $conf->{$key} = $value;
2475         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2476             my $value = $1;
2477             if ($section eq 'pending') {
2478                 $conf->{delete} = $value; # we parse this later
2479             } else {
2480                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2481             }
2482         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2483             my $key = $1;
2484             my $value = $2;
2485             eval { $value = check_type($key, $value); };
2486             if ($@) {
2487                 warn "vm $vmid - unable to parse value of '$key' - $@";
2488             } else {
2489                 $key = 'ide2' if $key eq 'cdrom';
2490                 my $fmt = $confdesc->{$key}->{format};
2491                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2492                     my $v = parse_drive($key, $value);
2493                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2494                         $v->{file} = $volid;
2495                         $value = print_drive($vmid, $v);
2496                     } else {
2497                         warn "vm $vmid - unable to parse value of '$key'\n";
2498                         next;
2499                     }
2500                 }
2501
2502                 $conf->{$key} = $value;
2503             }
2504         }
2505     }
2506
2507     if (defined($descr)) {
2508         $descr =~ s/\s+$//;
2509         $conf->{description} = $descr;
2510     }
2511     delete $res->{snapstate}; # just to be sure
2512
2513     return $res;
2514 }
2515
2516 sub write_vm_config {
2517     my ($filename, $conf) = @_;
2518
2519     delete $conf->{snapstate}; # just to be sure
2520
2521     if ($conf->{cdrom}) {
2522         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2523         $conf->{ide2} = $conf->{cdrom};
2524         delete $conf->{cdrom};
2525     }
2526
2527     # we do not use 'smp' any longer
2528     if ($conf->{sockets}) {
2529         delete $conf->{smp};
2530     } elsif ($conf->{smp}) {
2531         $conf->{sockets} = $conf->{smp};
2532         delete $conf->{cores};
2533         delete $conf->{smp};
2534     }
2535
2536     my $used_volids = {};
2537
2538     my $cleanup_config = sub {
2539         my ($cref, $pending, $snapname) = @_;
2540
2541         foreach my $key (keys %$cref) {
2542             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2543                 $key eq 'snapstate' || $key eq 'pending';
2544             my $value = $cref->{$key};
2545             if ($key eq 'delete') {
2546                 die "propertry 'delete' is only allowed in [PENDING]\n"
2547                     if !$pending;
2548                 # fixme: check syntax?
2549                 next;
2550             }
2551             eval { $value = check_type($key, $value); };
2552             die "unable to parse value of '$key' - $@" if $@;
2553
2554             $cref->{$key} = $value;
2555
2556             if (!$snapname && is_valid_drivename($key)) {
2557                 my $drive = parse_drive($key, $value);
2558                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2559             }
2560         }
2561     };
2562
2563     &$cleanup_config($conf);
2564
2565     &$cleanup_config($conf->{pending}, 1);
2566
2567     foreach my $snapname (keys %{$conf->{snapshots}}) {
2568         die "internal error" if $snapname eq 'pending';
2569         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2570     }
2571
2572     # remove 'unusedX' settings if we re-add a volume
2573     foreach my $key (keys %$conf) {
2574         my $value = $conf->{$key};
2575         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2576             delete $conf->{$key};
2577         }
2578     }
2579
2580     my $generate_raw_config = sub {
2581         my ($conf, $pending) = @_;
2582
2583         my $raw = '';
2584
2585         # add description as comment to top of file
2586         if (defined(my $descr = $conf->{description})) {
2587             if ($descr) {
2588                 foreach my $cl (split(/\n/, $descr)) {
2589                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2590                 }
2591             } else {
2592                 $raw .= "#\n" if $pending;
2593             }
2594         }
2595
2596         foreach my $key (sort keys %$conf) {
2597             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2598             $raw .= "$key: $conf->{$key}\n";
2599         }
2600         return $raw;
2601     };
2602
2603     my $raw = &$generate_raw_config($conf);
2604
2605     if (scalar(keys %{$conf->{pending}})){
2606         $raw .= "\n[PENDING]\n";
2607         $raw .= &$generate_raw_config($conf->{pending}, 1);
2608     }
2609
2610     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2611         $raw .= "\n[$snapname]\n";
2612         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2613     }
2614
2615     return $raw;
2616 }
2617
2618 sub load_defaults {
2619
2620     my $res = {};
2621
2622     # we use static defaults from our JSON schema configuration
2623     foreach my $key (keys %$confdesc) {
2624         if (defined(my $default = $confdesc->{$key}->{default})) {
2625             $res->{$key} = $default;
2626         }
2627     }
2628
2629     return $res;
2630 }
2631
2632 sub config_list {
2633     my $vmlist = PVE::Cluster::get_vmlist();
2634     my $res = {};
2635     return $res if !$vmlist || !$vmlist->{ids};
2636     my $ids = $vmlist->{ids};
2637
2638     foreach my $vmid (keys %$ids) {
2639         my $d = $ids->{$vmid};
2640         next if !$d->{node} || $d->{node} ne $nodename;
2641         next if !$d->{type} || $d->{type} ne 'qemu';
2642         $res->{$vmid}->{exists} = 1;
2643     }
2644     return $res;
2645 }
2646
2647 # test if VM uses local resources (to prevent migration)
2648 sub check_local_resources {
2649     my ($conf, $noerr) = @_;
2650
2651     my $loc_res = 0;
2652
2653     $loc_res = 1 if $conf->{hostusb}; # old syntax
2654     $loc_res = 1 if $conf->{hostpci}; # old syntax
2655
2656     foreach my $k (keys %$conf) {
2657         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2658         # sockets are safe: they will recreated be on the target side post-migrate
2659         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2660         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2661     }
2662
2663     die "VM uses local resources\n" if $loc_res && !$noerr;
2664
2665     return $loc_res;
2666 }
2667
2668 # check if used storages are available on all nodes (use by migrate)
2669 sub check_storage_availability {
2670     my ($storecfg, $conf, $node) = @_;
2671
2672     foreach_drive($conf, sub {
2673         my ($ds, $drive) = @_;
2674
2675         my $volid = $drive->{file};
2676         return if !$volid;
2677
2678         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2679         return if !$sid;
2680
2681         # check if storage is available on both nodes
2682         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2683         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2684    });
2685 }
2686
2687 # list nodes where all VM images are available (used by has_feature API)
2688 sub shared_nodes {
2689     my ($conf, $storecfg) = @_;
2690
2691     my $nodelist = PVE::Cluster::get_nodelist();
2692     my $nodehash = { map { $_ => 1 } @$nodelist };
2693     my $nodename = PVE::INotify::nodename();
2694
2695     foreach_drive($conf, sub {
2696         my ($ds, $drive) = @_;
2697
2698         my $volid = $drive->{file};
2699         return if !$volid;
2700
2701         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2702         if ($storeid) {
2703             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2704             if ($scfg->{disable}) {
2705                 $nodehash = {};
2706             } elsif (my $avail = $scfg->{nodes}) {
2707                 foreach my $node (keys %$nodehash) {
2708                     delete $nodehash->{$node} if !$avail->{$node};
2709                 }
2710             } elsif (!$scfg->{shared}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     delete $nodehash->{$node} if $node ne $nodename
2713                 }
2714             }
2715         }
2716     });
2717
2718     return $nodehash
2719 }
2720
2721 sub check_cmdline {
2722     my ($pidfile, $pid) = @_;
2723
2724     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2725     if (defined($fh)) {
2726         my $line = <$fh>;
2727         $fh->close;
2728         return undef if !$line;
2729         my @param = split(/\0/, $line);
2730
2731         my $cmd = $param[0];
2732         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2733
2734         for (my $i = 0; $i < scalar (@param); $i++) {
2735             my $p = $param[$i];
2736             next if !$p;
2737             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2738                 my $p = $param[$i+1];
2739                 return 1 if $p && ($p eq $pidfile);
2740                 return undef;
2741             }
2742         }
2743     }
2744     return undef;
2745 }
2746
2747 sub check_running {
2748     my ($vmid, $nocheck, $node) = @_;
2749
2750     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2751
2752     die "unable to find configuration file for VM $vmid - no such machine\n"
2753         if !$nocheck && ! -f $filename;
2754
2755     my $pidfile = pidfile_name($vmid);
2756
2757     if (my $fd = IO::File->new("<$pidfile")) {
2758         my $st = stat($fd);
2759         my $line = <$fd>;
2760         close($fd);
2761
2762         my $mtime = $st->mtime;
2763         if ($mtime > time()) {
2764             warn "file '$filename' modified in future\n";
2765         }
2766
2767         if ($line =~ m/^(\d+)$/) {
2768             my $pid = $1;
2769             if (check_cmdline($pidfile, $pid)) {
2770                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2771                     return $pid;
2772                 }
2773             }
2774         }
2775     }
2776
2777     return undef;
2778 }
2779
2780 sub vzlist {
2781
2782     my $vzlist = config_list();
2783
2784     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2785
2786     while (defined(my $de = $fd->read)) {
2787         next if $de !~ m/^(\d+)\.pid$/;
2788         my $vmid = $1;
2789         next if !defined($vzlist->{$vmid});
2790         if (my $pid = check_running($vmid)) {
2791             $vzlist->{$vmid}->{pid} = $pid;
2792         }
2793     }
2794
2795     return $vzlist;
2796 }
2797
2798 sub disksize {
2799     my ($storecfg, $conf) = @_;
2800
2801     my $bootdisk = $conf->{bootdisk};
2802     return undef if !$bootdisk;
2803     return undef if !is_valid_drivename($bootdisk);
2804
2805     return undef if !$conf->{$bootdisk};
2806
2807     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2808     return undef if !defined($drive);
2809
2810     return undef if drive_is_cdrom($drive);
2811
2812     my $volid = $drive->{file};
2813     return undef if !$volid;
2814
2815     return $drive->{size};
2816 }
2817
2818 our $vmstatus_return_properties = {
2819     vmid => get_standard_option('pve-vmid'),
2820     status => {
2821         description => "Qemu process status.",
2822         type => 'string',
2823         enum => ['stopped', 'running'],
2824     },
2825     maxmem => {
2826         description => "Maximum memory in bytes.",
2827         type => 'integer',
2828         optional => 1,
2829         renderer => 'bytes',
2830     },
2831     maxdisk => {
2832         description => "Root disk size in bytes.",
2833         type => 'integer',
2834         optional => 1,
2835         renderer => 'bytes',
2836     },
2837     name => {
2838         description => "VM name.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     qmpstatus => {
2843         description => "Qemu QMP agent status.",
2844         type => 'string',
2845         optional => 1,
2846     },
2847     pid => {
2848         description => "PID of running qemu process.",
2849         type => 'integer',
2850         optional => 1,
2851     },
2852     uptime => {
2853         description => "Uptime.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'duration',
2857     },
2858     cpus => {
2859         description => "Maximum usable CPUs.",
2860         type => 'number',
2861         optional => 1,
2862     },
2863 };
2864
2865 my $last_proc_pid_stat;
2866
2867 # get VM status information
2868 # This must be fast and should not block ($full == false)
2869 # We only query KVM using QMP if $full == true (this can be slow)
2870 sub vmstatus {
2871     my ($opt_vmid, $full) = @_;
2872
2873     my $res = {};
2874
2875     my $storecfg = PVE::Storage::config();
2876
2877     my $list = vzlist();
2878     my $defaults = load_defaults();
2879
2880     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2881
2882     my $cpucount = $cpuinfo->{cpus} || 1;
2883
2884     foreach my $vmid (keys %$list) {
2885         next if $opt_vmid && ($vmid ne $opt_vmid);
2886
2887         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2888         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2889
2890         my $d = { vmid => $vmid };
2891         $d->{pid} = $list->{$vmid}->{pid};
2892
2893         # fixme: better status?
2894         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2895
2896         my $size = disksize($storecfg, $conf);
2897         if (defined($size)) {
2898             $d->{disk} = 0; # no info available
2899             $d->{maxdisk} = $size;
2900         } else {
2901             $d->{disk} = 0;
2902             $d->{maxdisk} = 0;
2903         }
2904
2905         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2906             * ($conf->{cores} || $defaults->{cores});
2907         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2908         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2909
2910         $d->{name} = $conf->{name} || "VM $vmid";
2911         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2912             : $defaults->{memory}*(1024*1024);
2913
2914         if ($conf->{balloon}) {
2915             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2916             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2917                 : $defaults->{shares};
2918         }
2919
2920         $d->{uptime} = 0;
2921         $d->{cpu} = 0;
2922         $d->{mem} = 0;
2923
2924         $d->{netout} = 0;
2925         $d->{netin} = 0;
2926
2927         $d->{diskread} = 0;
2928         $d->{diskwrite} = 0;
2929
2930         $d->{template} = PVE::QemuConfig->is_template($conf);
2931
2932         $d->{serial} = 1 if conf_has_serial($conf);
2933
2934         $res->{$vmid} = $d;
2935     }
2936
2937     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2938     foreach my $dev (keys %$netdev) {
2939         next if $dev !~ m/^tap([1-9]\d*)i/;
2940         my $vmid = $1;
2941         my $d = $res->{$vmid};
2942         next if !$d;
2943
2944         $d->{netout} += $netdev->{$dev}->{receive};
2945         $d->{netin} += $netdev->{$dev}->{transmit};
2946
2947         if ($full) {
2948             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2949             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2950         }
2951
2952     }
2953
2954     my $ctime = gettimeofday;
2955
2956     foreach my $vmid (keys %$list) {
2957
2958         my $d = $res->{$vmid};
2959         my $pid = $d->{pid};
2960         next if !$pid;
2961
2962         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2963         next if !$pstat; # not running
2964
2965         my $used = $pstat->{utime} + $pstat->{stime};
2966
2967         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2968
2969         if ($pstat->{vsize}) {
2970             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2971         }
2972
2973         my $old = $last_proc_pid_stat->{$pid};
2974         if (!$old) {
2975             $last_proc_pid_stat->{$pid} = {
2976                 time => $ctime,
2977                 used => $used,
2978                 cpu => 0,
2979             };
2980             next;
2981         }
2982
2983         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2984
2985         if ($dtime > 1000) {
2986             my $dutime = $used -  $old->{used};
2987
2988             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2989             $last_proc_pid_stat->{$pid} = {
2990                 time => $ctime,
2991                 used => $used,
2992                 cpu => $d->{cpu},
2993             };
2994         } else {
2995             $d->{cpu} = $old->{cpu};
2996         }
2997     }
2998
2999     return $res if !$full;
3000
3001     my $qmpclient = PVE::QMPClient->new();
3002
3003     my $ballooncb = sub {
3004         my ($vmid, $resp) = @_;
3005
3006         my $info = $resp->{'return'};
3007         return if !$info->{max_mem};
3008
3009         my $d = $res->{$vmid};
3010
3011         # use memory assigned to VM
3012         $d->{maxmem} = $info->{max_mem};
3013         $d->{balloon} = $info->{actual};
3014
3015         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3016             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3017             $d->{freemem} = $info->{free_mem};
3018         }
3019
3020         $d->{ballooninfo} = $info;
3021     };
3022
3023     my $blockstatscb = sub {
3024         my ($vmid, $resp) = @_;
3025         my $data = $resp->{'return'} || [];
3026         my $totalrdbytes = 0;
3027         my $totalwrbytes = 0;
3028
3029         for my $blockstat (@$data) {
3030             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3031             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3032
3033             $blockstat->{device} =~ s/drive-//;
3034             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3035         }
3036         $res->{$vmid}->{diskread} = $totalrdbytes;
3037         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3038     };
3039
3040     my $statuscb = sub {
3041         my ($vmid, $resp) = @_;
3042
3043         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3044         # this fails if ballon driver is not loaded, so this must be
3045         # the last commnand (following command are aborted if this fails).
3046         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3047
3048         my $status = 'unknown';
3049         if (!defined($status = $resp->{'return'}->{status})) {
3050             warn "unable to get VM status\n";
3051             return;
3052         }
3053
3054         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3055     };
3056
3057     foreach my $vmid (keys %$list) {
3058         next if $opt_vmid && ($vmid ne $opt_vmid);
3059         next if !$res->{$vmid}->{pid}; # not running
3060         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3061     }
3062
3063     $qmpclient->queue_execute(undef, 2);
3064
3065     foreach my $vmid (keys %$list) {
3066         next if $opt_vmid && ($vmid ne $opt_vmid);
3067         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3068     }
3069
3070     return $res;
3071 }
3072
3073 sub foreach_drive {
3074     my ($conf, $func, @param) = @_;
3075
3076     foreach my $ds (valid_drive_names()) {
3077         next if !defined($conf->{$ds});
3078
3079         my $drive = parse_drive($ds, $conf->{$ds});
3080         next if !$drive;
3081
3082         &$func($ds, $drive, @param);
3083     }
3084 }
3085
3086 sub foreach_volid {
3087     my ($conf, $func, @param) = @_;
3088
3089     my $volhash = {};
3090
3091     my $test_volid = sub {
3092         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3093
3094         return if !$volid;
3095
3096         $volhash->{$volid}->{cdrom} //= 1;
3097         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3098
3099         $volhash->{$volid}->{replicate} //= 0;
3100         $volhash->{$volid}->{replicate} = 1 if $replicate;
3101
3102         $volhash->{$volid}->{shared} //= 0;
3103         $volhash->{$volid}->{shared} = 1 if $shared;
3104
3105         $volhash->{$volid}->{referenced_in_config} //= 0;
3106         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3107
3108         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3109             if defined($snapname);
3110     };
3111
3112     foreach_drive($conf, sub {
3113         my ($ds, $drive) = @_;
3114         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3115     });
3116
3117     foreach my $snapname (keys %{$conf->{snapshots}}) {
3118         my $snap = $conf->{snapshots}->{$snapname};
3119         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3120         foreach_drive($snap, sub {
3121             my ($ds, $drive) = @_;
3122             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3123         });
3124     }
3125
3126     foreach my $volid (keys %$volhash) {
3127         &$func($volid, $volhash->{$volid}, @param);
3128     }
3129 }
3130
3131 sub conf_has_serial {
3132     my ($conf) = @_;
3133
3134     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3135         if ($conf->{"serial$i"}) {
3136             return 1;
3137         }
3138     }
3139
3140     return 0;
3141 }
3142
3143 sub vga_conf_has_spice {
3144     my ($vga) = @_;
3145
3146     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3147
3148     return $1 || 1;
3149 }
3150
3151 sub config_to_command {
3152     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3153
3154     my $cmd = [];
3155     my $globalFlags = [];
3156     my $machineFlags = [];
3157     my $rtcFlags = [];
3158     my $cpuFlags = [];
3159     my $devices = [];
3160     my $pciaddr = '';
3161     my $bridges = {};
3162     my $kvmver = kvm_user_version();
3163     my $vernum = 0; # unknown
3164     my $ostype = $conf->{ostype};
3165     my $winversion = windows_version($ostype);
3166     my $kvm = $conf->{kvm} // 1;
3167
3168     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3169
3170     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3171         $vernum = $1*1000000+$2*1000;
3172     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3173         $vernum = $1*1000000+$2*1000+$3;
3174     }
3175
3176     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3177
3178     my $have_ovz = -f '/proc/vz/vestat';
3179
3180     my $q35 = machine_type_is_q35($conf);
3181     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3182     my $machine_type = $forcemachine || $conf->{machine};
3183     my $use_old_bios_files = undef;
3184     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3185
3186     my $cpuunits = defined($conf->{cpuunits}) ?
3187             $conf->{cpuunits} : $defaults->{cpuunits};
3188
3189     push @$cmd, '/usr/bin/kvm';
3190
3191     push @$cmd, '-id', $vmid;
3192
3193     my $vmname = $conf->{name} || "vm$vmid";
3194
3195     push @$cmd, '-name', $vmname;
3196
3197     my $use_virtio = 0;
3198
3199     my $qmpsocket = qmp_socket($vmid);
3200     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3201     push @$cmd, '-mon', "chardev=qmp,mode=control";
3202
3203
3204     push @$cmd, '-pidfile' , pidfile_name($vmid);
3205
3206     push @$cmd, '-daemonize';
3207
3208     if ($conf->{smbios1}) {
3209         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3210     }
3211
3212     if ($conf->{vmgenid}) {
3213         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3214     }
3215
3216     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3217         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3218
3219         my $path;
3220         my $format;
3221         if (my $efidisk = $conf->{efidisk0}) {
3222             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3223             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3224             $format = $d->{format};
3225             if ($storeid) {
3226                 $path = PVE::Storage::path($storecfg, $d->{file});
3227                 if (!defined($format)) {
3228                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3229                     $format = qemu_img_format($scfg, $volname);
3230                 }
3231             } else {
3232                 $path = $d->{file};
3233                 die "efidisk format must be specified\n"
3234                     if !defined($format);
3235             }
3236         } else {
3237             warn "no efidisk configured! Using temporary efivars disk.\n";
3238             $path = "/tmp/$vmid-ovmf.fd";
3239             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3240             $format = 'raw';
3241         }
3242
3243         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3244         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3245     }
3246
3247
3248     # add usb controllers
3249     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3250     push @$devices, @usbcontrollers if @usbcontrollers;
3251     my $vga = $conf->{vga};
3252
3253     my $qxlnum = vga_conf_has_spice($vga);
3254     $vga = 'qxl' if $qxlnum;
3255
3256     if (!$vga) {
3257         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3258             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3259         } else {
3260             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3261         }
3262     }
3263
3264     # enable absolute mouse coordinates (needed by vnc)
3265     my $tablet;
3266     if (defined($conf->{tablet})) {
3267         $tablet = $conf->{tablet};
3268     } else {
3269         $tablet = $defaults->{tablet};
3270         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3271         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3272     }
3273
3274     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3275
3276     my $kvm_off = 0;
3277     my $gpu_passthrough;
3278
3279     # host pci devices
3280     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3281         my $d = parse_hostpci($conf->{"hostpci$i"});
3282         next if !$d;
3283
3284         my $pcie = $d->{pcie};
3285         if($pcie){
3286             die "q35 machine model is not enabled" if !$q35;
3287             $pciaddr = print_pcie_addr("hostpci$i");
3288         }else{
3289             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3290         }
3291
3292         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3293         my $romfile = $d->{romfile};
3294
3295         my $xvga = '';
3296         if ($d->{'x-vga'}) {
3297             $xvga = ',x-vga=on';
3298             $kvm_off = 1;
3299             $vga = 'none';
3300             $gpu_passthrough = 1;
3301
3302             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3303                 $xvga = "";
3304             }
3305         }
3306         my $pcidevices = $d->{pciid};
3307         my $multifunction = 1 if @$pcidevices > 1;
3308
3309         my $j=0;
3310         foreach my $pcidevice (@$pcidevices) {
3311
3312             my $id = "hostpci$i";
3313             $id .= ".$j" if $multifunction;
3314             my $addr = $pciaddr;
3315             $addr .= ".$j" if $multifunction;
3316             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3317
3318             if($j == 0){
3319                 $devicestr .= "$rombar$xvga";
3320                 $devicestr .= ",multifunction=on" if $multifunction;
3321                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3322             }
3323
3324             push @$devices, '-device', $devicestr;
3325             $j++;
3326         }
3327     }
3328
3329     # usb devices
3330     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3331     push @$devices, @usbdevices if @usbdevices;
3332     # serial devices
3333     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3334         if (my $path = $conf->{"serial$i"}) {
3335             if ($path eq 'socket') {
3336                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3337                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3338                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3339             } else {
3340                 die "no such serial device\n" if ! -c $path;
3341                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3342                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3343             }
3344         }
3345     }
3346
3347     # parallel devices
3348     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3349         if (my $path = $conf->{"parallel$i"}) {
3350             die "no such parallel device\n" if ! -c $path;
3351             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3352             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3353             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3354         }
3355     }
3356
3357
3358     my $sockets = 1;
3359     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3360     $sockets = $conf->{sockets} if  $conf->{sockets};
3361
3362     my $cores = $conf->{cores} || 1;
3363
3364     my $maxcpus = $sockets * $cores;
3365
3366     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3367
3368     my $allowed_vcpus = $cpuinfo->{cpus};
3369
3370     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3371         if ($allowed_vcpus < $maxcpus);
3372
3373     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3374
3375         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3376         for (my $i = 2; $i <= $vcpus; $i++)  {
3377             my $cpustr = print_cpu_device($conf,$i);
3378             push @$cmd, '-device', $cpustr;
3379         }
3380
3381     } else {
3382
3383         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3384     }
3385     push @$cmd, '-nodefaults';
3386
3387     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3388
3389     my $bootindex_hash = {};
3390     my $i = 1;
3391     foreach my $o (split(//, $bootorder)) {
3392         $bootindex_hash->{$o} = $i*100;
3393         $i++;
3394     }
3395
3396     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3397
3398     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3399
3400     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3401
3402     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3403
3404     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3405         my $socket = vnc_socket($vmid);
3406         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3407     } else {
3408         push @$cmd, '-nographic';
3409     }
3410
3411     # time drift fix
3412     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3413
3414     my $useLocaltime = $conf->{localtime};
3415
3416     if ($winversion >= 5) { # windows
3417         $useLocaltime = 1 if !defined($conf->{localtime});
3418
3419         # use time drift fix when acpi is enabled
3420         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3421             $tdf = 1 if !defined($conf->{tdf});
3422         }
3423     }
3424
3425     if ($winversion >= 6) {
3426         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3427         push @$cmd, '-no-hpet';
3428     }
3429
3430     push @$rtcFlags, 'driftfix=slew' if $tdf;
3431
3432     if (!$kvm) {
3433         push @$machineFlags, 'accel=tcg';
3434     }
3435
3436     if ($machine_type) {
3437         push @$machineFlags, "type=${machine_type}";
3438     }
3439
3440     if ($conf->{startdate}) {
3441         push @$rtcFlags, "base=$conf->{startdate}";
3442     } elsif ($useLocaltime) {
3443         push @$rtcFlags, 'base=localtime';
3444     }
3445
3446     my $cpu = $kvm ? "kvm64" : "qemu64";
3447     if (my $cputype = $conf->{cpu}) {
3448         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3449             or die "Cannot parse cpu description: $cputype\n";
3450         $cpu = $cpuconf->{cputype};
3451         $kvm_off = 1 if $cpuconf->{hidden};
3452
3453         if (defined(my $flags = $cpuconf->{flags})) {
3454             push @$cpuFlags, split(";", $flags);
3455         }
3456     }
3457
3458     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3459
3460     push @$cpuFlags , '-x2apic'
3461         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3462
3463     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3464
3465     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3466
3467     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3468
3469         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3470         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3471     }
3472
3473     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3474
3475     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3476
3477     push @$cpuFlags, 'kvm=off' if $kvm_off;
3478
3479     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3480         die "internal error"; # should not happen
3481
3482     push @$cpuFlags, "vendor=${cpu_vendor}"
3483         if $cpu_vendor ne 'default';
3484
3485     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3486
3487     push @$cmd, '-cpu', $cpu;
3488
3489     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3490
3491     push @$cmd, '-S' if $conf->{freeze};
3492
3493     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3494
3495     # enable sound
3496     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3497     #push @$cmd, '-soundhw', 'es1370';
3498     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3499
3500     if (parse_guest_agent($conf)->{enabled}) {
3501         my $qgasocket = qmp_socket($vmid, 1);
3502         my $pciaddr = print_pci_addr("qga0", $bridges);
3503         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3504         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3505         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3506     }
3507
3508     my $spice_port;
3509
3510     if ($qxlnum) {
3511         if ($qxlnum > 1) {
3512             if ($winversion){
3513                 for(my $i = 1; $i < $qxlnum; $i++){
3514                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3515                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3516                 }
3517             } else {
3518                 # assume other OS works like Linux
3519                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3520                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3521             }
3522         }
3523
3524         my $pciaddr = print_pci_addr("spice", $bridges);
3525
3526         my $nodename = PVE::INotify::nodename();
3527         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3528         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3529         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3530         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3531         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3532
3533         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3534
3535         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3536         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3537         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3538     }
3539
3540     # enable balloon by default, unless explicitly disabled
3541     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3542         $pciaddr = print_pci_addr("balloon0", $bridges);
3543         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3544     }
3545
3546     if ($conf->{watchdog}) {
3547         my $wdopts = parse_watchdog($conf->{watchdog});
3548         $pciaddr = print_pci_addr("watchdog", $bridges);
3549         my $watchdog = $wdopts->{model} || 'i6300esb';
3550         push @$devices, '-device', "$watchdog$pciaddr";
3551         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3552     }
3553
3554     my $vollist = [];
3555     my $scsicontroller = {};
3556     my $ahcicontroller = {};
3557     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3558
3559     # Add iscsi initiator name if available
3560     if (my $initiator = get_initiator_name()) {
3561         push @$devices, '-iscsi', "initiator-name=$initiator";
3562     }
3563
3564     foreach_drive($conf, sub {
3565         my ($ds, $drive) = @_;
3566
3567         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3568             push @$vollist, $drive->{file};
3569         }
3570
3571         # ignore efidisk here, already added in bios/fw handling code above
3572         return if $drive->{interface} eq 'efidisk';
3573
3574         $use_virtio = 1 if $ds =~ m/^virtio/;
3575
3576         if (drive_is_cdrom ($drive)) {
3577             if ($bootindex_hash->{d}) {
3578                 $drive->{bootindex} = $bootindex_hash->{d};
3579                 $bootindex_hash->{d} += 1;
3580             }
3581         } else {
3582             if ($bootindex_hash->{c}) {
3583                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3584                 $bootindex_hash->{c} += 1;
3585             }
3586         }
3587
3588         if($drive->{interface} eq 'virtio'){
3589            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3590         }
3591
3592         if ($drive->{interface} eq 'scsi') {
3593
3594             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3595
3596             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3597             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3598
3599             my $iothread = '';
3600             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3601                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3602                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3603             } elsif ($drive->{iothread}) {
3604                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3605             }
3606
3607             my $queues = '';
3608             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3609                 $queues = ",num_queues=$drive->{queues}";
3610             }
3611
3612             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3613             $scsicontroller->{$controller}=1;
3614         }
3615
3616         if ($drive->{interface} eq 'sata') {
3617            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3618            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3619            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3620            $ahcicontroller->{$controller}=1;
3621         }
3622
3623         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3624         push @$devices, '-drive',$drive_cmd;
3625         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3626     });
3627
3628     for (my $i = 0; $i < $MAX_NETS; $i++) {
3629          next if !$conf->{"net$i"};
3630          my $d = parse_net($conf->{"net$i"});
3631          next if !$d;
3632
3633          $use_virtio = 1 if $d->{model} eq 'virtio';
3634
3635          if ($bootindex_hash->{n}) {
3636             $d->{bootindex} = $bootindex_hash->{n};
3637             $bootindex_hash->{n} += 1;
3638          }
3639
3640          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3641          push @$devices, '-netdev', $netdevfull;
3642
3643          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3644          push @$devices, '-device', $netdevicefull;
3645     }
3646
3647     if (!$q35) {
3648         # add pci bridges
3649         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3650            $bridges->{1} = 1;
3651            $bridges->{2} = 1;
3652         }
3653
3654         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3655
3656         while (my ($k, $v) = each %$bridges) {
3657             $pciaddr = print_pci_addr("pci.$k");
3658             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3659         }
3660     }
3661
3662     # add custom args
3663     if ($conf->{args}) {
3664         my $aa = PVE::Tools::split_args($conf->{args});
3665         push @$cmd, @$aa;
3666     }
3667
3668     push @$cmd, @$devices;
3669     push @$cmd, '-rtc', join(',', @$rtcFlags)
3670         if scalar(@$rtcFlags);
3671     push @$cmd, '-machine', join(',', @$machineFlags)
3672         if scalar(@$machineFlags);
3673     push @$cmd, '-global', join(',', @$globalFlags)
3674         if scalar(@$globalFlags);
3675
3676     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3677 }
3678
3679 sub vnc_socket {
3680     my ($vmid) = @_;
3681     return "${var_run_tmpdir}/$vmid.vnc";
3682 }
3683
3684 sub spice_port {
3685     my ($vmid) = @_;
3686
3687     my $res = vm_mon_cmd($vmid, 'query-spice');
3688
3689     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3690 }
3691
3692 sub qmp_socket {
3693     my ($vmid, $qga) = @_;
3694     my $sockettype = $qga ? 'qga' : 'qmp';
3695     return "${var_run_tmpdir}/$vmid.$sockettype";
3696 }
3697
3698 sub pidfile_name {
3699     my ($vmid) = @_;
3700     return "${var_run_tmpdir}/$vmid.pid";
3701 }
3702
3703 sub vm_devices_list {
3704     my ($vmid) = @_;
3705
3706     my $res = vm_mon_cmd($vmid, 'query-pci');
3707     my $devices_to_check = [];
3708     my $devices = {};
3709     foreach my $pcibus (@$res) {
3710         push @$devices_to_check, @{$pcibus->{devices}},
3711     }
3712
3713     while (@$devices_to_check) {
3714         my $to_check = [];
3715         for my $d (@$devices_to_check) {
3716             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3717             next if !$d->{'pci_bridge'};
3718
3719             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3720             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3721         }
3722         $devices_to_check = $to_check;
3723     }
3724
3725     my $resblock = vm_mon_cmd($vmid, 'query-block');
3726     foreach my $block (@$resblock) {
3727         if($block->{device} =~ m/^drive-(\S+)/){
3728                 $devices->{$1} = 1;
3729         }
3730     }
3731
3732     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3733     foreach my $mice (@$resmice) {
3734         if ($mice->{name} eq 'QEMU HID Tablet') {
3735             $devices->{tablet} = 1;
3736             last;
3737         }
3738     }
3739
3740     # for usb devices there is no query-usb
3741     # but we can iterate over the entries in
3742     # qom-list path=/machine/peripheral
3743     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3744     foreach my $per (@$resperipheral) {
3745         if ($per->{name} =~ m/^usb\d+$/) {
3746             $devices->{$per->{name}} = 1;
3747         }
3748     }
3749
3750     return $devices;
3751 }
3752
3753 sub vm_deviceplug {
3754     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3755
3756     my $q35 = machine_type_is_q35($conf);
3757
3758     my $devices_list = vm_devices_list($vmid);
3759     return 1 if defined($devices_list->{$deviceid});
3760
3761     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3762
3763     if ($deviceid eq 'tablet') {
3764
3765         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3766
3767     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3768
3769         die "usb hotplug currently not reliable\n";
3770         # since we can't reliably hot unplug all added usb devices
3771         # and usb passthrough disables live migration
3772         # we disable usb hotplugging for now
3773         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3774
3775     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3776
3777         qemu_iothread_add($vmid, $deviceid, $device);
3778
3779         qemu_driveadd($storecfg, $vmid, $device);
3780         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3781
3782         qemu_deviceadd($vmid, $devicefull);
3783         eval { qemu_deviceaddverify($vmid, $deviceid); };
3784         if (my $err = $@) {
3785             eval { qemu_drivedel($vmid, $deviceid); };
3786             warn $@ if $@;
3787             die $err;
3788         }
3789
3790     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3791
3792
3793         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3794         my $pciaddr = print_pci_addr($deviceid);
3795         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3796
3797         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3798
3799         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3800             qemu_iothread_add($vmid, $deviceid, $device);
3801             $devicefull .= ",iothread=iothread-$deviceid";
3802         }
3803
3804         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3805             $devicefull .= ",num_queues=$device->{queues}";
3806         }
3807
3808         qemu_deviceadd($vmid, $devicefull);
3809         qemu_deviceaddverify($vmid, $deviceid);
3810
3811     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3812
3813         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3814         qemu_driveadd($storecfg, $vmid, $device);
3815
3816         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3817         eval { qemu_deviceadd($vmid, $devicefull); };
3818         if (my $err = $@) {
3819             eval { qemu_drivedel($vmid, $deviceid); };
3820             warn $@ if $@;
3821             die $err;
3822         }
3823
3824     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3825
3826         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3827
3828         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3829         my $use_old_bios_files = undef;
3830         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3831
3832         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3833         qemu_deviceadd($vmid, $netdevicefull);
3834         eval { qemu_deviceaddverify($vmid, $deviceid); };
3835         if (my $err = $@) {
3836             eval { qemu_netdevdel($vmid, $deviceid); };
3837             warn $@ if $@;
3838             die $err;
3839         }
3840
3841     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3842
3843         my $bridgeid = $2;
3844         my $pciaddr = print_pci_addr($deviceid);
3845         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3846
3847         qemu_deviceadd($vmid, $devicefull);
3848         qemu_deviceaddverify($vmid, $deviceid);
3849
3850     } else {
3851         die "can't hotplug device '$deviceid'\n";
3852     }
3853
3854     return 1;
3855 }
3856
3857 # fixme: this should raise exceptions on error!
3858 sub vm_deviceunplug {
3859     my ($vmid, $conf, $deviceid) = @_;
3860
3861     my $devices_list = vm_devices_list($vmid);
3862     return 1 if !defined($devices_list->{$deviceid});
3863
3864     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3865
3866     if ($deviceid eq 'tablet') {
3867
3868         qemu_devicedel($vmid, $deviceid);
3869
3870     } elsif ($deviceid =~ m/^usb\d+$/) {
3871
3872         die "usb hotplug currently not reliable\n";
3873         # when unplugging usb devices this way,
3874         # there may be remaining usb controllers/hubs
3875         # so we disable it for now
3876         qemu_devicedel($vmid, $deviceid);
3877         qemu_devicedelverify($vmid, $deviceid);
3878
3879     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3880
3881         qemu_devicedel($vmid, $deviceid);
3882         qemu_devicedelverify($vmid, $deviceid);
3883         qemu_drivedel($vmid, $deviceid);
3884         qemu_iothread_del($conf, $vmid, $deviceid);
3885
3886     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3887
3888         qemu_devicedel($vmid, $deviceid);
3889         qemu_devicedelverify($vmid, $deviceid);
3890         qemu_iothread_del($conf, $vmid, $deviceid);
3891
3892     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3893
3894         qemu_devicedel($vmid, $deviceid);
3895         qemu_drivedel($vmid, $deviceid);
3896         qemu_deletescsihw($conf, $vmid, $deviceid);
3897
3898     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3899
3900         qemu_devicedel($vmid, $deviceid);
3901         qemu_devicedelverify($vmid, $deviceid);
3902         qemu_netdevdel($vmid, $deviceid);
3903
3904     } else {
3905         die "can't unplug device '$deviceid'\n";
3906     }
3907
3908     return 1;
3909 }
3910
3911 sub qemu_deviceadd {
3912     my ($vmid, $devicefull) = @_;
3913
3914     $devicefull = "driver=".$devicefull;
3915     my %options =  split(/[=,]/, $devicefull);
3916
3917     vm_mon_cmd($vmid, "device_add" , %options);
3918 }
3919
3920 sub qemu_devicedel {
3921     my ($vmid, $deviceid) = @_;
3922
3923     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3924 }
3925
3926 sub qemu_iothread_add {
3927     my($vmid, $deviceid, $device) = @_;
3928
3929     if ($device->{iothread}) {
3930         my $iothreads = vm_iothreads_list($vmid);
3931         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3932     }
3933 }
3934
3935 sub qemu_iothread_del {
3936     my($conf, $vmid, $deviceid) = @_;
3937
3938     my $device = parse_drive($deviceid, $conf->{$deviceid});
3939     if ($device->{iothread}) {
3940         my $iothreads = vm_iothreads_list($vmid);
3941         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3942     }
3943 }
3944
3945 sub qemu_objectadd {
3946     my($vmid, $objectid, $qomtype) = @_;
3947
3948     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3949
3950     return 1;
3951 }
3952
3953 sub qemu_objectdel {
3954     my($vmid, $objectid) = @_;
3955
3956     vm_mon_cmd($vmid, "object-del", id => $objectid);
3957
3958     return 1;
3959 }
3960
3961 sub qemu_driveadd {
3962     my ($storecfg, $vmid, $device) = @_;
3963
3964     my $drive = print_drive_full($storecfg, $vmid, $device);
3965     $drive =~ s/\\/\\\\/g;
3966     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3967
3968     # If the command succeeds qemu prints: "OK"
3969     return 1 if $ret =~ m/OK/s;
3970
3971     die "adding drive failed: $ret\n";
3972 }
3973
3974 sub qemu_drivedel {
3975     my($vmid, $deviceid) = @_;
3976
3977     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3978     $ret =~ s/^\s+//;
3979
3980     return 1 if $ret eq "";
3981
3982     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3983     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3984
3985     die "deleting drive $deviceid failed : $ret\n";
3986 }
3987
3988 sub qemu_deviceaddverify {
3989     my ($vmid, $deviceid) = @_;
3990
3991     for (my $i = 0; $i <= 5; $i++) {
3992          my $devices_list = vm_devices_list($vmid);
3993          return 1 if defined($devices_list->{$deviceid});
3994          sleep 1;
3995     }
3996
3997     die "error on hotplug device '$deviceid'\n";
3998 }
3999
4000
4001 sub qemu_devicedelverify {
4002     my ($vmid, $deviceid) = @_;
4003
4004     # need to verify that the device is correctly removed as device_del
4005     # is async and empty return is not reliable
4006
4007     for (my $i = 0; $i <= 5; $i++) {
4008          my $devices_list = vm_devices_list($vmid);
4009          return 1 if !defined($devices_list->{$deviceid});
4010          sleep 1;
4011     }
4012
4013     die "error on hot-unplugging device '$deviceid'\n";
4014 }
4015
4016 sub qemu_findorcreatescsihw {
4017     my ($storecfg, $conf, $vmid, $device) = @_;
4018
4019     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4020
4021     my $scsihwid="$controller_prefix$controller";
4022     my $devices_list = vm_devices_list($vmid);
4023
4024     if(!defined($devices_list->{$scsihwid})) {
4025         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
4026     }
4027
4028     return 1;
4029 }
4030
4031 sub qemu_deletescsihw {
4032     my ($conf, $vmid, $opt) = @_;
4033
4034     my $device = parse_drive($opt, $conf->{$opt});
4035
4036     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4037         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4038         return 1;
4039     }
4040
4041     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4042
4043     my $devices_list = vm_devices_list($vmid);
4044     foreach my $opt (keys %{$devices_list}) {
4045         if (PVE::QemuServer::is_valid_drivename($opt)) {
4046             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4047             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4048                 return 1;
4049             }
4050         }
4051     }
4052
4053     my $scsihwid="scsihw$controller";
4054
4055     vm_deviceunplug($vmid, $conf, $scsihwid);
4056
4057     return 1;
4058 }
4059
4060 sub qemu_add_pci_bridge {
4061     my ($storecfg, $conf, $vmid, $device) = @_;
4062
4063     my $bridges = {};
4064
4065     my $bridgeid;
4066
4067     print_pci_addr($device, $bridges);
4068
4069     while (my ($k, $v) = each %$bridges) {
4070         $bridgeid = $k;
4071     }
4072     return 1 if !defined($bridgeid) || $bridgeid < 1;
4073
4074     my $bridge = "pci.$bridgeid";
4075     my $devices_list = vm_devices_list($vmid);
4076
4077     if (!defined($devices_list->{$bridge})) {
4078         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4079     }
4080
4081     return 1;
4082 }
4083
4084 sub qemu_set_link_status {
4085     my ($vmid, $device, $up) = @_;
4086
4087     vm_mon_cmd($vmid, "set_link", name => $device,
4088                up => $up ? JSON::true : JSON::false);
4089 }
4090
4091 sub qemu_netdevadd {
4092     my ($vmid, $conf, $device, $deviceid) = @_;
4093
4094     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4095     my %options =  split(/[=,]/, $netdev);
4096
4097     vm_mon_cmd($vmid, "netdev_add",  %options);
4098     return 1;
4099 }
4100
4101 sub qemu_netdevdel {
4102     my ($vmid, $deviceid) = @_;
4103
4104     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4105 }
4106
4107 sub qemu_usb_hotplug {
4108     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4109
4110     return if !$device;
4111
4112     # remove the old one first
4113     vm_deviceunplug($vmid, $conf, $deviceid);
4114
4115     # check if xhci controller is necessary and available
4116     if ($device->{usb3}) {
4117
4118         my $devicelist = vm_devices_list($vmid);
4119
4120         if (!$devicelist->{xhci}) {
4121             my $pciaddr = print_pci_addr("xhci");
4122             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4123         }
4124     }
4125     my $d = parse_usb_device($device->{host});
4126     $d->{usb3} = $device->{usb3};
4127
4128     # add the new one
4129     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4130 }
4131
4132 sub qemu_cpu_hotplug {
4133     my ($vmid, $conf, $vcpus) = @_;
4134
4135     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4136
4137     my $sockets = 1;
4138     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4139     $sockets = $conf->{sockets} if  $conf->{sockets};
4140     my $cores = $conf->{cores} || 1;
4141     my $maxcpus = $sockets * $cores;
4142
4143     $vcpus = $maxcpus if !$vcpus;
4144
4145     die "you can't add more vcpus than maxcpus\n"
4146         if $vcpus > $maxcpus;
4147
4148     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4149
4150     if ($vcpus < $currentvcpus) {
4151
4152         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4153
4154             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4155                 qemu_devicedel($vmid, "cpu$i");
4156                 my $retry = 0;
4157                 my $currentrunningvcpus = undef;
4158                 while (1) {
4159                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4160                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4161                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4162                     $retry++;
4163                     sleep 1;
4164                 }
4165                 #update conf after each succesfull cpu unplug
4166                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4167                 PVE::QemuConfig->write_config($vmid, $conf);
4168             }
4169         } else {
4170             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4171         }
4172
4173         return;
4174     }
4175
4176     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4177     die "vcpus in running vm does not match its configuration\n"
4178         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4179
4180     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4181
4182         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4183             my $cpustr = print_cpu_device($conf, $i);
4184             qemu_deviceadd($vmid, $cpustr);
4185
4186             my $retry = 0;
4187             my $currentrunningvcpus = undef;
4188             while (1) {
4189                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4190                 last if scalar(@{$currentrunningvcpus}) == $i;
4191                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4192                 sleep 1;
4193                 $retry++;
4194             }
4195             #update conf after each succesfull cpu hotplug
4196             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4197             PVE::QemuConfig->write_config($vmid, $conf);
4198         }
4199     } else {
4200
4201         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4202             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4203         }
4204     }
4205 }
4206
4207 sub qemu_block_set_io_throttle {
4208     my ($vmid, $deviceid,
4209         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4210         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4211         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4212         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4213
4214     return if !check_running($vmid) ;
4215
4216     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4217         bps => int($bps),
4218         bps_rd => int($bps_rd),
4219         bps_wr => int($bps_wr),
4220         iops => int($iops),
4221         iops_rd => int($iops_rd),
4222         iops_wr => int($iops_wr),
4223         bps_max => int($bps_max),
4224         bps_rd_max => int($bps_rd_max),
4225         bps_wr_max => int($bps_wr_max),
4226         iops_max => int($iops_max),
4227         iops_rd_max => int($iops_rd_max),
4228         iops_wr_max => int($iops_wr_max),
4229         bps_max_length => int($bps_max_length),
4230         bps_rd_max_length => int($bps_rd_max_length),
4231         bps_wr_max_length => int($bps_wr_max_length),
4232         iops_max_length => int($iops_max_length),
4233         iops_rd_max_length => int($iops_rd_max_length),
4234         iops_wr_max_length => int($iops_wr_max_length),
4235     );
4236
4237 }
4238
4239 # old code, only used to shutdown old VM after update
4240 sub __read_avail {
4241     my ($fh, $timeout) = @_;
4242
4243     my $sel = new IO::Select;
4244     $sel->add($fh);
4245
4246     my $res = '';
4247     my $buf;
4248
4249     my @ready;
4250     while (scalar (@ready = $sel->can_read($timeout))) {
4251         my $count;
4252         if ($count = $fh->sysread($buf, 8192)) {
4253             if ($buf =~ /^(.*)\(qemu\) $/s) {
4254                 $res .= $1;
4255                 last;
4256             } else {
4257                 $res .= $buf;
4258             }
4259         } else {
4260             if (!defined($count)) {
4261                 die "$!\n";
4262             }
4263             last;
4264         }
4265     }
4266
4267     die "monitor read timeout\n" if !scalar(@ready);
4268
4269     return $res;
4270 }
4271
4272 sub qemu_block_resize {
4273     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4274
4275     my $running = check_running($vmid);
4276
4277     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4278
4279     return if !$running;
4280
4281     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4282
4283 }
4284
4285 sub qemu_volume_snapshot {
4286     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4287
4288     my $running = check_running($vmid);
4289
4290     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4291         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4292     } else {
4293         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4294     }
4295 }
4296
4297 sub qemu_volume_snapshot_delete {
4298     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4299
4300     my $running = check_running($vmid);
4301
4302     if($running) {
4303
4304         $running = undef;
4305         my $conf = PVE::QemuConfig->load_config($vmid);
4306         foreach_drive($conf, sub {
4307             my ($ds, $drive) = @_;
4308             $running = 1 if $drive->{file} eq $volid;
4309         });
4310     }
4311
4312     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4313         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4314     } else {
4315         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4316     }
4317 }
4318
4319 sub set_migration_caps {
4320     my ($vmid) = @_;
4321
4322     my $cap_ref = [];
4323
4324     my $enabled_cap = {
4325         "auto-converge" => 1,
4326         "xbzrle" => 1,
4327         "x-rdma-pin-all" => 0,
4328         "zero-blocks" => 0,
4329         "compress" => 0
4330     };
4331
4332     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4333
4334     for my $supported_capability (@$supported_capabilities) {
4335         push @$cap_ref, {
4336             capability => $supported_capability->{capability},
4337             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4338         };
4339     }
4340
4341     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4342 }
4343
4344 my $fast_plug_option = {
4345     'lock' => 1,
4346     'name' => 1,
4347     'onboot' => 1,
4348     'shares' => 1,
4349     'startup' => 1,
4350     'description' => 1,
4351     'protection' => 1,
4352     'vmstatestorage' => 1,
4353 };
4354
4355 # hotplug changes in [PENDING]
4356 # $selection hash can be used to only apply specified options, for
4357 # example: { cores => 1 } (only apply changed 'cores')
4358 # $errors ref is used to return error messages
4359 sub vmconfig_hotplug_pending {
4360     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4361
4362     my $defaults = load_defaults();
4363
4364     # commit values which do not have any impact on running VM first
4365     # Note: those option cannot raise errors, we we do not care about
4366     # $selection and always apply them.
4367
4368     my $add_error = sub {
4369         my ($opt, $msg) = @_;
4370         $errors->{$opt} = "hotplug problem - $msg";
4371     };
4372
4373     my $changes = 0;
4374     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4375         if ($fast_plug_option->{$opt}) {
4376             $conf->{$opt} = $conf->{pending}->{$opt};
4377             delete $conf->{pending}->{$opt};
4378             $changes = 1;
4379         }
4380     }
4381
4382     if ($changes) {
4383         PVE::QemuConfig->write_config($vmid, $conf);
4384         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4385     }
4386
4387     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4388
4389     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4390     while (my ($opt, $force) = each %$pending_delete_hash) {
4391         next if $selection && !$selection->{$opt};
4392         eval {
4393             if ($opt eq 'hotplug') {
4394                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4395             } elsif ($opt eq 'tablet') {
4396                 die "skip\n" if !$hotplug_features->{usb};
4397                 if ($defaults->{tablet}) {
4398                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4399                 } else {
4400                     vm_deviceunplug($vmid, $conf, $opt);
4401                 }
4402             } elsif ($opt =~ m/^usb\d+/) {
4403                 die "skip\n";
4404                 # since we cannot reliably hot unplug usb devices
4405                 # we are disabling it
4406                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4407                 vm_deviceunplug($vmid, $conf, $opt);
4408             } elsif ($opt eq 'vcpus') {
4409                 die "skip\n" if !$hotplug_features->{cpu};
4410                 qemu_cpu_hotplug($vmid, $conf, undef);
4411             } elsif ($opt eq 'balloon') {
4412                 # enable balloon device is not hotpluggable
4413                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4414                 # here we reset the ballooning value to memory
4415                 my $balloon = $conf->{memory} || $defaults->{memory};
4416                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4417             } elsif ($fast_plug_option->{$opt}) {
4418                 # do nothing
4419             } elsif ($opt =~ m/^net(\d+)$/) {
4420                 die "skip\n" if !$hotplug_features->{network};
4421                 vm_deviceunplug($vmid, $conf, $opt);
4422             } elsif (is_valid_drivename($opt)) {
4423                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4424                 vm_deviceunplug($vmid, $conf, $opt);
4425                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4426             } elsif ($opt =~ m/^memory$/) {
4427                 die "skip\n" if !$hotplug_features->{memory};
4428                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4429             } elsif ($opt eq 'cpuunits') {
4430                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4431             } elsif ($opt eq 'cpulimit') {
4432                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4433             } else {
4434                 die "skip\n";
4435             }
4436         };
4437         if (my $err = $@) {
4438             &$add_error($opt, $err) if $err ne "skip\n";
4439         } else {
4440             # save new config if hotplug was successful
4441             delete $conf->{$opt};
4442             vmconfig_undelete_pending_option($conf, $opt);
4443             PVE::QemuConfig->write_config($vmid, $conf);
4444             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4445         }
4446     }
4447
4448     my $apply_pending_cloudinit;
4449     $apply_pending_cloudinit = sub {
4450         my ($key, $value) = @_;
4451         $apply_pending_cloudinit = sub {}; # once is enough
4452
4453         my @cloudinit_opts = keys %$confdesc_cloudinit;
4454         foreach my $opt (keys %{$conf->{pending}}) {
4455             next if !grep { $_ eq $opt } @cloudinit_opts;
4456             $conf->{$opt} = delete $conf->{pending}->{$opt};
4457         }
4458
4459         my $new_conf = { %$conf };
4460         $new_conf->{$key} = $value;
4461         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4462     };
4463
4464     foreach my $opt (keys %{$conf->{pending}}) {
4465         next if $selection && !$selection->{$opt};
4466         my $value = $conf->{pending}->{$opt};
4467         eval {
4468             if ($opt eq 'hotplug') {
4469                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4470             } elsif ($opt eq 'tablet') {
4471                 die "skip\n" if !$hotplug_features->{usb};
4472                 if ($value == 1) {
4473                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4474                 } elsif ($value == 0) {
4475                     vm_deviceunplug($vmid, $conf, $opt);
4476                 }
4477             } elsif ($opt =~ m/^usb\d+$/) {
4478                 die "skip\n";
4479                 # since we cannot reliably hot unplug usb devices
4480                 # we are disabling it
4481                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4482                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4483                 die "skip\n" if !$d;
4484                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4485             } elsif ($opt eq 'vcpus') {
4486                 die "skip\n" if !$hotplug_features->{cpu};
4487                 qemu_cpu_hotplug($vmid, $conf, $value);
4488             } elsif ($opt eq 'balloon') {
4489                 # enable/disable balloning device is not hotpluggable
4490                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4491                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4492                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4493
4494                 # allow manual ballooning if shares is set to zero
4495                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4496                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4497                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4498                 }
4499             } elsif ($opt =~ m/^net(\d+)$/) {
4500                 # some changes can be done without hotplug
4501                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4502                                     $vmid, $opt, $value);
4503             } elsif (is_valid_drivename($opt)) {
4504                 # some changes can be done without hotplug
4505                 my $drive = parse_drive($opt, $value);
4506                 if (drive_is_cloudinit($drive)) {
4507                     &$apply_pending_cloudinit($opt, $value);
4508                 }
4509                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4510                                      $vmid, $opt, $value, 1);
4511             } elsif ($opt =~ m/^memory$/) { #dimms
4512                 die "skip\n" if !$hotplug_features->{memory};
4513                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4514             } elsif ($opt eq 'cpuunits') {
4515                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4516             } elsif ($opt eq 'cpulimit') {
4517                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4518                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4519             } else {
4520                 die "skip\n";  # skip non-hot-pluggable options
4521             }
4522         };
4523         if (my $err = $@) {
4524             &$add_error($opt, $err) if $err ne "skip\n";
4525         } else {
4526             # save new config if hotplug was successful
4527             $conf->{$opt} = $value;
4528             delete $conf->{pending}->{$opt};
4529             PVE::QemuConfig->write_config($vmid, $conf);
4530             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4531         }
4532     }
4533 }
4534
4535 sub try_deallocate_drive {
4536     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4537
4538     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4539         my $volid = $drive->{file};
4540         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4541             my $sid = PVE::Storage::parse_volume_id($volid);
4542             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4543
4544             # check if the disk is really unused
4545             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4546                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4547             PVE::Storage::vdisk_free($storecfg, $volid);
4548             return 1;
4549         } else {
4550             # If vm is not owner of this disk remove from config
4551             return 1;
4552         }
4553     }
4554
4555     return undef;
4556 }
4557
4558 sub vmconfig_delete_or_detach_drive {
4559     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4560
4561     my $drive = parse_drive($opt, $conf->{$opt});
4562
4563     my $rpcenv = PVE::RPCEnvironment::get();
4564     my $authuser = $rpcenv->get_user();
4565
4566     if ($force) {
4567         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4568         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4569     } else {
4570         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4571     }
4572 }
4573
4574 sub vmconfig_apply_pending {
4575     my ($vmid, $conf, $storecfg) = @_;
4576
4577     # cold plug
4578
4579     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4580     while (my ($opt, $force) = each %$pending_delete_hash) {
4581         die "internal error" if $opt =~ m/^unused/;
4582         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4583         if (!defined($conf->{$opt})) {
4584             vmconfig_undelete_pending_option($conf, $opt);
4585             PVE::QemuConfig->write_config($vmid, $conf);
4586         } elsif (is_valid_drivename($opt)) {
4587             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4588             vmconfig_undelete_pending_option($conf, $opt);
4589             delete $conf->{$opt};
4590             PVE::QemuConfig->write_config($vmid, $conf);
4591         } else {
4592             vmconfig_undelete_pending_option($conf, $opt);
4593             delete $conf->{$opt};
4594             PVE::QemuConfig->write_config($vmid, $conf);
4595         }
4596     }
4597
4598     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4599
4600     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4601         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4602
4603         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4604             # skip if nothing changed
4605         } elsif (is_valid_drivename($opt)) {
4606             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4607                 if defined($conf->{$opt});
4608             $conf->{$opt} = $conf->{pending}->{$opt};
4609         } else {
4610             $conf->{$opt} = $conf->{pending}->{$opt};
4611         }
4612
4613         delete $conf->{pending}->{$opt};
4614         PVE::QemuConfig->write_config($vmid, $conf);
4615     }
4616 }
4617
4618 my $safe_num_ne = sub {
4619     my ($a, $b) = @_;
4620
4621     return 0 if !defined($a) && !defined($b);
4622     return 1 if !defined($a);
4623     return 1 if !defined($b);
4624
4625     return $a != $b;
4626 };
4627
4628 my $safe_string_ne = sub {
4629     my ($a, $b) = @_;
4630
4631     return 0 if !defined($a) && !defined($b);
4632     return 1 if !defined($a);
4633     return 1 if !defined($b);
4634
4635     return $a ne $b;
4636 };
4637
4638 sub vmconfig_update_net {
4639     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4640
4641     my $newnet = parse_net($value);
4642
4643     if ($conf->{$opt}) {
4644         my $oldnet = parse_net($conf->{$opt});
4645
4646         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4647             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4648             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4649             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4650
4651             # for non online change, we try to hot-unplug
4652             die "skip\n" if !$hotplug;
4653             vm_deviceunplug($vmid, $conf, $opt);
4654         } else {
4655
4656             die "internal error" if $opt !~ m/net(\d+)/;
4657             my $iface = "tap${vmid}i$1";
4658
4659             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4660                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4661                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4662                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4663                 PVE::Network::tap_unplug($iface);
4664                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4665             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4666                 # Rate can be applied on its own but any change above needs to
4667                 # include the rate in tap_plug since OVS resets everything.
4668                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4669             }
4670
4671             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4672                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4673             }
4674
4675             return 1;
4676         }
4677     }
4678
4679     if ($hotplug) {
4680         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4681     } else {
4682         die "skip\n";
4683     }
4684 }
4685
4686 sub vmconfig_update_disk {
4687     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4688
4689     # fixme: do we need force?
4690
4691     my $drive = parse_drive($opt, $value);
4692
4693     if ($conf->{$opt}) {
4694
4695         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4696
4697             my $media = $drive->{media} || 'disk';
4698             my $oldmedia = $old_drive->{media} || 'disk';
4699             die "unable to change media type\n" if $media ne $oldmedia;
4700
4701             if (!drive_is_cdrom($old_drive)) {
4702
4703                 if ($drive->{file} ne $old_drive->{file}) {
4704
4705                     die "skip\n" if !$hotplug;
4706
4707                     # unplug and register as unused
4708                     vm_deviceunplug($vmid, $conf, $opt);
4709                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4710
4711                 } else {
4712                     # update existing disk
4713
4714                     # skip non hotpluggable value
4715                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4716                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4717                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4718                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4719                         die "skip\n";
4720                     }
4721
4722                     # apply throttle
4723                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4724                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4725                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4726                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4727                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4728                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4729                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4730                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4731                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4732                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4733                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4734                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4735                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4736                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4737                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4738                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4739                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4740                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4741
4742                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4743                                                    ($drive->{mbps} || 0)*1024*1024,
4744                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4745                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4746                                                    $drive->{iops} || 0,
4747                                                    $drive->{iops_rd} || 0,
4748                                                    $drive->{iops_wr} || 0,
4749                                                    ($drive->{mbps_max} || 0)*1024*1024,
4750                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4751                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4752                                                    $drive->{iops_max} || 0,
4753                                                    $drive->{iops_rd_max} || 0,
4754                                                    $drive->{iops_wr_max} || 0,
4755                                                    $drive->{bps_max_length} || 1,
4756                                                    $drive->{bps_rd_max_length} || 1,
4757                                                    $drive->{bps_wr_max_length} || 1,
4758                                                    $drive->{iops_max_length} || 1,
4759                                                    $drive->{iops_rd_max_length} || 1,
4760                                                    $drive->{iops_wr_max_length} || 1);
4761
4762                     }
4763
4764                     return 1;
4765                 }
4766
4767             } else { # cdrom
4768
4769                 if ($drive->{file} eq 'none') {
4770                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4771                     if (drive_is_cloudinit($old_drive)) {
4772                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4773                     }
4774                 } else {
4775                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4776                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4777                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4778                 }
4779
4780                 return 1;
4781             }
4782         }
4783     }
4784
4785     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4786     # hotplug new disks
4787     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4788     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4789 }
4790
4791 sub vm_start {
4792     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4793         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4794
4795     PVE::QemuConfig->lock_config($vmid, sub {
4796         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4797
4798         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4799
4800         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4801
4802         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4803
4804         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4805             vmconfig_apply_pending($vmid, $conf, $storecfg);
4806             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4807         }
4808
4809         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4810
4811         my $defaults = load_defaults();
4812
4813         # set environment variable useful inside network script
4814         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4815
4816         my $local_volumes = {};
4817
4818         if ($targetstorage) {
4819             foreach_drive($conf, sub {
4820                 my ($ds, $drive) = @_;
4821
4822                 return if drive_is_cdrom($drive);
4823
4824                 my $volid = $drive->{file};
4825
4826                 return if !$volid;
4827
4828                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4829
4830                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4831                 return if $scfg->{shared};
4832                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4833             });
4834
4835             my $format = undef;
4836
4837             foreach my $opt (sort keys %$local_volumes) {
4838
4839                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4840                 my $drive = parse_drive($opt, $conf->{$opt});
4841
4842                 #if remote storage is specified, use default format
4843                 if ($targetstorage && $targetstorage ne "1") {
4844                     $storeid = $targetstorage;
4845                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4846                     $format = $defFormat;
4847                 } else {
4848                     #else we use same format than original
4849                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4850                     $format = qemu_img_format($scfg, $volid);
4851                 }
4852
4853                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4854                 my $newdrive = $drive;
4855                 $newdrive->{format} = $format;
4856                 $newdrive->{file} = $newvolid;
4857                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4858                 $local_volumes->{$opt} = $drivestr;
4859                 #pass drive to conf for command line
4860                 $conf->{$opt} = $drivestr;
4861             }
4862         }
4863
4864         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4865
4866         my $migrate_port = 0;
4867         my $migrate_uri;
4868         if ($statefile) {
4869             if ($statefile eq 'tcp') {
4870                 my $localip = "localhost";
4871                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4872                 my $nodename = PVE::INotify::nodename();
4873
4874                 if (!defined($migration_type)) {
4875                     if (defined($datacenterconf->{migration}->{type})) {
4876                         $migration_type = $datacenterconf->{migration}->{type};
4877                     } else {
4878                         $migration_type = 'secure';
4879                     }
4880                 }
4881
4882                 if ($migration_type eq 'insecure') {
4883                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4884                     if ($migrate_network_addr) {
4885                         $localip = $migrate_network_addr;
4886                     } else {
4887                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4888                     }
4889
4890                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4891                 }
4892
4893                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4894                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4895                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4896                 push @$cmd, '-incoming', $migrate_uri;
4897                 push @$cmd, '-S';
4898
4899             } elsif ($statefile eq 'unix') {
4900                 # should be default for secure migrations as a ssh TCP forward
4901                 # tunnel is not deterministic reliable ready and fails regurarly
4902                 # to set up in time, so use UNIX socket forwards
4903                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4904                 unlink $socket_addr;
4905
4906                 $migrate_uri = "unix:$socket_addr";
4907
4908                 push @$cmd, '-incoming', $migrate_uri;
4909                 push @$cmd, '-S';
4910
4911             } else {
4912                 push @$cmd, '-loadstate', $statefile;
4913             }
4914         } elsif ($paused) {
4915             push @$cmd, '-S';
4916         }
4917
4918         # host pci devices
4919         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4920           my $d = parse_hostpci($conf->{"hostpci$i"});
4921           next if !$d;
4922           my $pcidevices = $d->{pciid};
4923           foreach my $pcidevice (@$pcidevices) {
4924                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4925
4926                 my $info = pci_device_info("0000:$pciid");
4927                 die "IOMMU not present\n" if !check_iommu_support();
4928                 die "no pci device info for device '$pciid'\n" if !$info;
4929                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4930                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4931           }
4932         }
4933
4934         PVE::Storage::activate_volumes($storecfg, $vollist);
4935
4936         if (!check_running($vmid, 1)) {
4937             eval {
4938                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4939                     outfunc => sub {}, errfunc => sub {});
4940             };
4941         }
4942
4943         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4944                                                   : $defaults->{cpuunits};
4945
4946         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4947         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4948
4949         my %properties = (
4950             Slice => 'qemu.slice',
4951             KillMode => 'none',
4952             CPUShares => $cpuunits
4953         );
4954
4955         if (my $cpulimit = $conf->{cpulimit}) {
4956             $properties{CPUQuota} = int($cpulimit * 100);
4957         }
4958         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4959
4960         my $run_qemu = sub {
4961             PVE::Tools::run_fork sub {
4962                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4963                 run_command($cmd, %run_params);
4964             };
4965         };
4966
4967         if ($conf->{hugepages}) {
4968
4969             my $code = sub {
4970                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4971                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4972
4973                 PVE::QemuServer::Memory::hugepages_mount();
4974                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4975
4976                 eval { $run_qemu->() };
4977                 if (my $err = $@) {
4978                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4979                     die $err;
4980                 }
4981
4982                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4983             };
4984             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4985
4986         } else {
4987             eval { $run_qemu->() };
4988         }
4989
4990         if (my $err = $@) {
4991             # deactivate volumes if start fails
4992             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4993             die "start failed: $err";
4994         }
4995
4996         print "migration listens on $migrate_uri\n" if $migrate_uri;
4997
4998         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4999             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5000             warn $@ if $@;
5001         }
5002
5003         #start nbd server for storage migration
5004         if ($targetstorage) {
5005             my $nodename = PVE::INotify::nodename();
5006             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5007             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5008             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5009             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5010
5011             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5012
5013             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5014
5015             foreach my $opt (sort keys %$local_volumes) {
5016                 my $volid = $local_volumes->{$opt};
5017                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5018                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5019                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5020             }
5021         }
5022
5023         if ($migratedfrom) {
5024             eval {
5025                 set_migration_caps($vmid);
5026             };
5027             warn $@ if $@;
5028
5029             if ($spice_port) {
5030                 print "spice listens on port $spice_port\n";
5031                 if ($spice_ticket) {
5032                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5033                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5034                 }
5035             }
5036
5037         } else {
5038             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5039                 if !$statefile && $conf->{balloon};
5040
5041             foreach my $opt (keys %$conf) {
5042                 next if $opt !~  m/^net\d+$/;
5043                 my $nicconf = parse_net($conf->{$opt});
5044                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5045             }
5046         }
5047
5048         vm_mon_cmd_nocheck($vmid, 'qom-set',
5049                     path => "machine/peripheral/balloon0",
5050                     property => "guest-stats-polling-interval",
5051                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5052
5053     });
5054 }
5055
5056 sub vm_mon_cmd {
5057     my ($vmid, $execute, %params) = @_;
5058
5059     my $cmd = { execute => $execute, arguments => \%params };
5060     vm_qmp_command($vmid, $cmd);
5061 }
5062
5063 sub vm_mon_cmd_nocheck {
5064     my ($vmid, $execute, %params) = @_;
5065
5066     my $cmd = { execute => $execute, arguments => \%params };
5067     vm_qmp_command($vmid, $cmd, 1);
5068 }
5069
5070 sub vm_qmp_command {
5071     my ($vmid, $cmd, $nocheck) = @_;
5072
5073     my $res;
5074
5075     my $timeout;
5076     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5077         $timeout = $cmd->{arguments}->{timeout};
5078         delete $cmd->{arguments}->{timeout};
5079     }
5080
5081     eval {
5082         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5083         my $sname = qmp_socket($vmid);
5084         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5085             my $qmpclient = PVE::QMPClient->new();
5086
5087             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5088         } else {
5089             die "unable to open monitor socket\n";
5090         }
5091     };
5092     if (my $err = $@) {
5093         syslog("err", "VM $vmid qmp command failed - $err");
5094         die $err;
5095     }
5096
5097     return $res;
5098 }
5099
5100 sub vm_human_monitor_command {
5101     my ($vmid, $cmdline) = @_;
5102
5103     my $res;
5104
5105     my $cmd = {
5106         execute => 'human-monitor-command',
5107         arguments => { 'command-line' => $cmdline},
5108     };
5109
5110     return vm_qmp_command($vmid, $cmd);
5111 }
5112
5113 sub vm_commandline {
5114     my ($storecfg, $vmid) = @_;
5115
5116     my $conf = PVE::QemuConfig->load_config($vmid);
5117
5118     my $defaults = load_defaults();
5119
5120     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5121
5122     return PVE::Tools::cmd2string($cmd);
5123 }
5124
5125 sub vm_reset {
5126     my ($vmid, $skiplock) = @_;
5127
5128     PVE::QemuConfig->lock_config($vmid, sub {
5129
5130         my $conf = PVE::QemuConfig->load_config($vmid);
5131
5132         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5133
5134         vm_mon_cmd($vmid, "system_reset");
5135     });
5136 }
5137
5138 sub get_vm_volumes {
5139     my ($conf) = @_;
5140
5141     my $vollist = [];
5142     foreach_volid($conf, sub {
5143         my ($volid, $attr) = @_;
5144
5145         return if $volid =~ m|^/|;
5146
5147         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5148         return if !$sid;
5149
5150         push @$vollist, $volid;
5151     });
5152
5153     return $vollist;
5154 }
5155
5156 sub vm_stop_cleanup {
5157     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5158
5159     eval {
5160
5161         if (!$keepActive) {
5162             my $vollist = get_vm_volumes($conf);
5163             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5164         }
5165
5166         foreach my $ext (qw(mon qmp pid vnc qga)) {
5167             unlink "/var/run/qemu-server/${vmid}.$ext";
5168         }
5169
5170         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5171     };
5172     warn $@ if $@; # avoid errors - just warn
5173 }
5174
5175 # Note: use $nockeck to skip tests if VM configuration file exists.
5176 # We need that when migration VMs to other nodes (files already moved)
5177 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5178 sub vm_stop {
5179     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5180
5181     $force = 1 if !defined($force) && !$shutdown;
5182
5183     if ($migratedfrom){
5184         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5185         kill 15, $pid if $pid;
5186         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5187         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5188         return;
5189     }
5190
5191     PVE::QemuConfig->lock_config($vmid, sub {
5192
5193         my $pid = check_running($vmid, $nocheck);
5194         return if !$pid;
5195
5196         my $conf;
5197         if (!$nocheck) {
5198             $conf = PVE::QemuConfig->load_config($vmid);
5199             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5200             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5201                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5202                 $timeout = $opts->{down} if $opts->{down};
5203             }
5204         }
5205
5206         $timeout = 60 if !defined($timeout);
5207
5208         eval {
5209             if ($shutdown) {
5210                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5211                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5212                 } else {
5213                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5214                 }
5215             } else {
5216                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5217             }
5218         };
5219         my $err = $@;
5220
5221         if (!$err) {
5222             my $count = 0;
5223             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5224                 $count++;
5225                 sleep 1;
5226             }
5227
5228             if ($count >= $timeout) {
5229                 if ($force) {
5230                     warn "VM still running - terminating now with SIGTERM\n";
5231                     kill 15, $pid;
5232                 } else {
5233                     die "VM quit/powerdown failed - got timeout\n";
5234                 }
5235             } else {
5236                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5237                 return;
5238             }
5239         } else {
5240             if ($force) {
5241                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5242                 kill 15, $pid;
5243             } else {
5244                 die "VM quit/powerdown failed\n";
5245             }
5246         }
5247
5248         # wait again
5249         $timeout = 10;
5250
5251         my $count = 0;
5252         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5253             $count++;
5254             sleep 1;
5255         }
5256
5257         if ($count >= $timeout) {
5258             warn "VM still running - terminating now with SIGKILL\n";
5259             kill 9, $pid;
5260             sleep 1;
5261         }
5262
5263         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5264    });
5265 }
5266
5267 sub vm_suspend {
5268     my ($vmid, $skiplock) = @_;
5269
5270     PVE::QemuConfig->lock_config($vmid, sub {
5271
5272         my $conf = PVE::QemuConfig->load_config($vmid);
5273
5274         PVE::QemuConfig->check_lock($conf)
5275             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5276
5277         vm_mon_cmd($vmid, "stop");
5278     });
5279 }
5280
5281 sub vm_resume {
5282     my ($vmid, $skiplock, $nocheck) = @_;
5283
5284     PVE::QemuConfig->lock_config($vmid, sub {
5285
5286         my $res = vm_mon_cmd($vmid, 'query-status');
5287         my $resume_cmd = 'cont';
5288
5289         if ($res->{status} && $res->{status} eq 'suspended') {
5290             $resume_cmd = 'system_wakeup';
5291         }
5292
5293         if (!$nocheck) {
5294
5295             my $conf = PVE::QemuConfig->load_config($vmid);
5296
5297             PVE::QemuConfig->check_lock($conf)
5298                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5299
5300             vm_mon_cmd($vmid, $resume_cmd);
5301
5302         } else {
5303             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5304         }
5305     });
5306 }
5307
5308 sub vm_sendkey {
5309     my ($vmid, $skiplock, $key) = @_;
5310
5311     PVE::QemuConfig->lock_config($vmid, sub {
5312
5313         my $conf = PVE::QemuConfig->load_config($vmid);
5314
5315         # there is no qmp command, so we use the human monitor command
5316         vm_human_monitor_command($vmid, "sendkey $key");
5317     });
5318 }
5319
5320 sub vm_destroy {
5321     my ($storecfg, $vmid, $skiplock) = @_;
5322
5323     PVE::QemuConfig->lock_config($vmid, sub {
5324
5325         my $conf = PVE::QemuConfig->load_config($vmid);
5326
5327         if (!check_running($vmid)) {
5328             destroy_vm($storecfg, $vmid, undef, $skiplock);
5329         } else {
5330             die "VM $vmid is running - destroy failed\n";
5331         }
5332     });
5333 }
5334
5335 # pci helpers
5336
5337 sub file_write {
5338     my ($filename, $buf) = @_;
5339
5340     my $fh = IO::File->new($filename, "w");
5341     return undef if !$fh;
5342
5343     my $res = print $fh $buf;
5344
5345     $fh->close();
5346
5347     return $res;
5348 }
5349
5350 sub pci_device_info {
5351     my ($name) = @_;
5352
5353     my $res;
5354
5355     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5356     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5357
5358     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5359     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5360
5361     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5362     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5363
5364     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5365     return undef if !defined($product) || $product !~ s/^0x//;
5366
5367     $res = {
5368         name => $name,
5369         vendor => $vendor,
5370         product => $product,
5371         domain => $domain,
5372         bus => $bus,
5373         slot => $slot,
5374         func => $func,
5375         irq => $irq,
5376         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5377     };
5378
5379     return $res;
5380 }
5381
5382 sub pci_dev_reset {
5383     my ($dev) = @_;
5384
5385     my $name = $dev->{name};
5386
5387     my $fn = "$pcisysfs/devices/$name/reset";
5388
5389     return file_write($fn, "1");
5390 }
5391
5392 sub pci_dev_bind_to_vfio {
5393     my ($dev) = @_;
5394
5395     my $name = $dev->{name};
5396
5397     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5398
5399     if (!-d $vfio_basedir) {
5400         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5401     }
5402     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5403
5404     my $testdir = "$vfio_basedir/$name";
5405     return 1 if -d $testdir;
5406
5407     my $data = "$dev->{vendor} $dev->{product}";
5408     return undef if !file_write("$vfio_basedir/new_id", $data);
5409
5410     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5411     if (!file_write($fn, $name)) {
5412         return undef if -f $fn;
5413     }
5414
5415     $fn = "$vfio_basedir/bind";
5416     if (! -d $testdir) {
5417         return undef if !file_write($fn, $name);
5418     }
5419
5420     return -d $testdir;
5421 }
5422
5423 sub pci_dev_group_bind_to_vfio {
5424     my ($pciid) = @_;
5425
5426     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5427
5428     if (!-d $vfio_basedir) {
5429         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5430     }
5431     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5432
5433     # get IOMMU group devices
5434     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5435       my @devs = grep /^0000:/, readdir($D);
5436     closedir($D);
5437
5438     foreach my $pciid (@devs) {
5439         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5440
5441         # pci bridges, switches or root ports are not supported
5442         # they have a pci_bus subdirectory so skip them
5443         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5444
5445         my $info = pci_device_info($1);
5446         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5447     }
5448
5449     return 1;
5450 }
5451
5452 # vzdump restore implementaion
5453
5454 sub tar_archive_read_firstfile {
5455     my $archive = shift;
5456
5457     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5458
5459     # try to detect archive type first
5460     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5461         die "unable to open file '$archive'\n";
5462     my $firstfile = <$fh>;
5463     kill 15, $pid;
5464     close $fh;
5465
5466     die "ERROR: archive contaions no data\n" if !$firstfile;
5467     chomp $firstfile;
5468
5469     return $firstfile;
5470 }
5471
5472 sub tar_restore_cleanup {
5473     my ($storecfg, $statfile) = @_;
5474
5475     print STDERR "starting cleanup\n";
5476
5477     if (my $fd = IO::File->new($statfile, "r")) {
5478         while (defined(my $line = <$fd>)) {
5479             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5480                 my $volid = $2;
5481                 eval {
5482                     if ($volid =~ m|^/|) {
5483                         unlink $volid || die 'unlink failed\n';
5484                     } else {
5485                         PVE::Storage::vdisk_free($storecfg, $volid);
5486                     }
5487                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5488                 };
5489                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5490             } else {
5491                 print STDERR "unable to parse line in statfile - $line";
5492             }
5493         }
5494         $fd->close();
5495     }
5496 }
5497
5498 sub restore_archive {
5499     my ($archive, $vmid, $user, $opts) = @_;
5500
5501     my $format = $opts->{format};
5502     my $comp;
5503
5504     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5505         $format = 'tar' if !$format;
5506         $comp = 'gzip';
5507     } elsif ($archive =~ m/\.tar$/) {
5508         $format = 'tar' if !$format;
5509     } elsif ($archive =~ m/.tar.lzo$/) {
5510         $format = 'tar' if !$format;
5511         $comp = 'lzop';
5512     } elsif ($archive =~ m/\.vma$/) {
5513         $format = 'vma' if !$format;
5514     } elsif ($archive =~ m/\.vma\.gz$/) {
5515         $format = 'vma' if !$format;
5516         $comp = 'gzip';
5517     } elsif ($archive =~ m/\.vma\.lzo$/) {
5518         $format = 'vma' if !$format;
5519         $comp = 'lzop';
5520     } else {
5521         $format = 'vma' if !$format; # default
5522     }
5523
5524     # try to detect archive format
5525     if ($format eq 'tar') {
5526         return restore_tar_archive($archive, $vmid, $user, $opts);
5527     } else {
5528         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5529     }
5530 }
5531
5532 sub restore_update_config_line {
5533     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5534
5535     return if $line =~ m/^\#qmdump\#/;
5536     return if $line =~ m/^\#vzdump\#/;
5537     return if $line =~ m/^lock:/;
5538     return if $line =~ m/^unused\d+:/;
5539     return if $line =~ m/^parent:/;
5540     return if $line =~ m/^template:/; # restored VM is never a template
5541
5542     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5543     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5544         # try to convert old 1.X settings
5545         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5546         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5547             my ($model, $macaddr) = split(/\=/, $devconfig);
5548             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5549             my $net = {
5550                 model => $model,
5551                 bridge => "vmbr$ind",
5552                 macaddr => $macaddr,
5553             };
5554             my $netstr = print_net($net);
5555
5556             print $outfd "net$cookie->{netcount}: $netstr\n";
5557             $cookie->{netcount}++;
5558         }
5559     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5560         my ($id, $netstr) = ($1, $2);
5561         my $net = parse_net($netstr);
5562         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5563         $netstr = print_net($net);
5564         print $outfd "$id: $netstr\n";
5565     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5566         my $virtdev = $1;
5567         my $value = $3;
5568         my $di = parse_drive($virtdev, $value);
5569         if (defined($di->{backup}) && !$di->{backup}) {
5570             print $outfd "#$line";
5571         } elsif ($map->{$virtdev}) {
5572             delete $di->{format}; # format can change on restore
5573             $di->{file} = $map->{$virtdev};
5574             $value = print_drive($vmid, $di);
5575             print $outfd "$virtdev: $value\n";
5576         } else {
5577             print $outfd $line;
5578         }
5579     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5580         my $vmgenid = $1;
5581         if ($vmgenid ne '0') {
5582             # always generate a new vmgenid if there was a valid one setup
5583             $vmgenid = generate_uuid();
5584         }
5585         print $outfd "vmgenid: $vmgenid\n";
5586     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5587         my ($uuid, $uuid_str);
5588         UUID::generate($uuid);
5589         UUID::unparse($uuid, $uuid_str);
5590         my $smbios1 = parse_smbios1($2);
5591         $smbios1->{uuid} = $uuid_str;
5592         print $outfd $1.print_smbios1($smbios1)."\n";
5593     } else {
5594         print $outfd $line;
5595     }
5596 }
5597
5598 sub scan_volids {
5599     my ($cfg, $vmid) = @_;
5600
5601     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5602
5603     my $volid_hash = {};
5604     foreach my $storeid (keys %$info) {
5605         foreach my $item (@{$info->{$storeid}}) {
5606             next if !($item->{volid} && $item->{size});
5607             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5608             $volid_hash->{$item->{volid}} = $item;
5609         }
5610     }
5611
5612     return $volid_hash;
5613 }
5614
5615 sub is_volume_in_use {
5616     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5617
5618     my $path = PVE::Storage::path($storecfg, $volid);
5619
5620     my $scan_config = sub {
5621         my ($cref, $snapname) = @_;
5622
5623         foreach my $key (keys %$cref) {
5624             my $value = $cref->{$key};
5625             if (is_valid_drivename($key)) {
5626                 next if $skip_drive && $key eq $skip_drive;
5627                 my $drive = parse_drive($key, $value);
5628                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5629                 return 1 if $volid eq $drive->{file};
5630                 if ($drive->{file} =~ m!^/!) {
5631                     return 1 if $drive->{file} eq $path;
5632                 } else {
5633                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5634                     next if !$storeid;
5635                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5636                     next if !$scfg;
5637                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5638                 }
5639             }
5640         }
5641
5642         return 0;
5643     };
5644
5645     return 1 if &$scan_config($conf);
5646
5647     undef $skip_drive;
5648
5649     foreach my $snapname (keys %{$conf->{snapshots}}) {
5650         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5651     }
5652
5653     return 0;
5654 }
5655
5656 sub update_disksize {
5657     my ($vmid, $conf, $volid_hash) = @_;
5658
5659     my $changes;
5660     my $prefix = "VM $vmid:";
5661
5662     # used and unused disks
5663     my $referenced = {};
5664
5665     # Note: it is allowed to define multiple storages with same path (alias), so
5666     # we need to check both 'volid' and real 'path' (two different volid can point
5667     # to the same path).
5668
5669     my $referencedpath = {};
5670
5671     # update size info
5672     foreach my $opt (keys %$conf) {
5673         if (is_valid_drivename($opt)) {
5674             my $drive = parse_drive($opt, $conf->{$opt});
5675             my $volid = $drive->{file};
5676             next if !$volid;
5677
5678             $referenced->{$volid} = 1;
5679             if ($volid_hash->{$volid} &&
5680                 (my $path = $volid_hash->{$volid}->{path})) {
5681                 $referencedpath->{$path} = 1;
5682             }
5683
5684             next if drive_is_cdrom($drive);
5685             next if !$volid_hash->{$volid};
5686
5687             $drive->{size} = $volid_hash->{$volid}->{size};
5688             my $new = print_drive($vmid, $drive);
5689             if ($new ne $conf->{$opt}) {
5690                 $changes = 1;
5691                 $conf->{$opt} = $new;
5692                 print "$prefix update disk '$opt' information.\n";
5693             }
5694         }
5695     }
5696
5697     # remove 'unusedX' entry if volume is used
5698     foreach my $opt (keys %$conf) {
5699         next if $opt !~ m/^unused\d+$/;
5700         my $volid = $conf->{$opt};
5701         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5702         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5703             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5704             $changes = 1;
5705             delete $conf->{$opt};
5706         }
5707
5708         $referenced->{$volid} = 1;
5709         $referencedpath->{$path} = 1 if $path;
5710     }
5711
5712     foreach my $volid (sort keys %$volid_hash) {
5713         next if $volid =~ m/vm-$vmid-state-/;
5714         next if $referenced->{$volid};
5715         my $path = $volid_hash->{$volid}->{path};
5716         next if !$path; # just to be sure
5717         next if $referencedpath->{$path};
5718         $changes = 1;
5719         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5720         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5721         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5722     }
5723
5724     return $changes;
5725 }
5726
5727 sub rescan {
5728     my ($vmid, $nolock, $dryrun) = @_;
5729
5730     my $cfg = PVE::Storage::config();
5731
5732     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5733     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5734     foreach my $stor (keys %{$cfg->{ids}}) {
5735         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5736     }
5737
5738     print "rescan volumes...\n";
5739     my $volid_hash = scan_volids($cfg, $vmid);
5740
5741     my $updatefn =  sub {
5742         my ($vmid) = @_;
5743
5744         my $conf = PVE::QemuConfig->load_config($vmid);
5745
5746         PVE::QemuConfig->check_lock($conf);
5747
5748         my $vm_volids = {};
5749         foreach my $volid (keys %$volid_hash) {
5750             my $info = $volid_hash->{$volid};
5751             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5752         }
5753
5754         my $changes = update_disksize($vmid, $conf, $vm_volids);
5755
5756         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5757     };
5758
5759     if (defined($vmid)) {
5760         if ($nolock) {
5761             &$updatefn($vmid);
5762         } else {
5763             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5764         }
5765     } else {
5766         my $vmlist = config_list();
5767         foreach my $vmid (keys %$vmlist) {
5768             if ($nolock) {
5769                 &$updatefn($vmid);
5770             } else {
5771                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5772             }
5773         }
5774     }
5775 }
5776
5777 sub restore_vma_archive {
5778     my ($archive, $vmid, $user, $opts, $comp) = @_;
5779
5780     my $readfrom = $archive;
5781
5782     my $cfg = PVE::Storage::config();
5783     my $commands = [];
5784     my $bwlimit = $opts->{bwlimit};
5785
5786     my $dbg_cmdstring = '';
5787     my $add_pipe = sub {
5788         my ($cmd) = @_;
5789         push @$commands, $cmd;
5790         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5791         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5792         $readfrom = '-';
5793     };
5794
5795     my $input = undef;
5796     if ($archive eq '-') {
5797         $input = '<&STDIN';
5798     } else {
5799         # If we use a backup from a PVE defined storage we also consider that
5800         # storage's rate limit:
5801         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5802         if (defined($volid)) {
5803             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5804             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5805             if ($readlimit) {
5806                 print STDERR "applying read rate limit: $readlimit\n";
5807                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5808                 $add_pipe->($cstream);
5809             }
5810         }
5811     }
5812
5813     if ($comp) {
5814         my $cmd;
5815         if ($comp eq 'gzip') {
5816             $cmd = ['zcat', $readfrom];
5817         } elsif ($comp eq 'lzop') {
5818             $cmd = ['lzop', '-d', '-c', $readfrom];
5819         } else {
5820             die "unknown compression method '$comp'\n";
5821         }
5822         $add_pipe->($cmd);
5823     }
5824
5825     my $tmpdir = "/var/tmp/vzdumptmp$$";
5826     rmtree $tmpdir;
5827
5828     # disable interrupts (always do cleanups)
5829     local $SIG{INT} =
5830         local $SIG{TERM} =
5831         local $SIG{QUIT} =
5832         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5833
5834     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5835     POSIX::mkfifo($mapfifo, 0600);
5836     my $fifofh;
5837
5838     my $openfifo = sub {
5839         open($fifofh, '>', $mapfifo) || die $!;
5840     };
5841
5842     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5843
5844     my $oldtimeout;
5845     my $timeout = 5;
5846
5847     my $devinfo = {};
5848
5849     my $rpcenv = PVE::RPCEnvironment::get();
5850
5851     my $conffile = PVE::QemuConfig->config_file($vmid);
5852     my $tmpfn = "$conffile.$$.tmp";
5853
5854     # Note: $oldconf is undef if VM does not exists
5855     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5856     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5857
5858     my %storage_limits;
5859
5860     my $print_devmap = sub {
5861         my $virtdev_hash = {};
5862
5863         my $cfgfn = "$tmpdir/qemu-server.conf";
5864
5865         # we can read the config - that is already extracted
5866         my $fh = IO::File->new($cfgfn, "r") ||
5867             "unable to read qemu-server.conf - $!\n";
5868
5869         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5870         if (-f $fwcfgfn) {
5871             my $pve_firewall_dir = '/etc/pve/firewall';
5872             mkdir $pve_firewall_dir; # make sure the dir exists
5873             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5874         }
5875
5876         while (defined(my $line = <$fh>)) {
5877             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5878                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5879                 die "archive does not contain data for drive '$virtdev'\n"
5880                     if !$devinfo->{$devname};
5881                 if (defined($opts->{storage})) {
5882                     $storeid = $opts->{storage} || 'local';
5883                 } elsif (!$storeid) {
5884                     $storeid = 'local';
5885                 }
5886                 $format = 'raw' if !$format;
5887                 $devinfo->{$devname}->{devname} = $devname;
5888                 $devinfo->{$devname}->{virtdev} = $virtdev;
5889                 $devinfo->{$devname}->{format} = $format;
5890                 $devinfo->{$devname}->{storeid} = $storeid;
5891
5892                 # check permission on storage
5893                 my $pool = $opts->{pool}; # todo: do we need that?
5894                 if ($user ne 'root@pam') {
5895                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5896                 }
5897
5898                 $storage_limits{$storeid} = $bwlimit;
5899
5900                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5901             }
5902         }
5903
5904         foreach my $key (keys %storage_limits) {
5905             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5906             next if !$limit;
5907             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5908             $storage_limits{$key} = $limit * 1024;
5909         }
5910
5911         foreach my $devname (keys %$devinfo) {
5912             die "found no device mapping information for device '$devname'\n"
5913                 if !$devinfo->{$devname}->{virtdev};
5914         }
5915
5916         # create empty/temp config
5917         if ($oldconf) {
5918             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5919             foreach_drive($oldconf, sub {
5920                 my ($ds, $drive) = @_;
5921
5922                 return if drive_is_cdrom($drive);
5923
5924                 my $volid = $drive->{file};
5925
5926                 return if !$volid || $volid =~ m|^/|;
5927
5928                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5929                 return if !$path || !$owner || ($owner != $vmid);
5930
5931                 # Note: only delete disk we want to restore
5932                 # other volumes will become unused
5933                 if ($virtdev_hash->{$ds}) {
5934                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5935                     if (my $err = $@) {
5936                         warn $err;
5937                     }
5938                 }
5939             });
5940
5941             # delete vmstate files
5942             # since after the restore we have no snapshots anymore
5943             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5944                 my $snap = $oldconf->{snapshots}->{$snapname};
5945                 if ($snap->{vmstate}) {
5946                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5947                     if (my $err = $@) {
5948                         warn $err;
5949                     }
5950                 }
5951             }
5952         }
5953
5954         my $map = {};
5955         foreach my $virtdev (sort keys %$virtdev_hash) {
5956             my $d = $virtdev_hash->{$virtdev};
5957             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5958             my $storeid = $d->{storeid};
5959             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5960
5961             my $map_opts = '';
5962             if (my $limit = $storage_limits{$storeid}) {
5963                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5964             }
5965
5966             # test if requested format is supported
5967             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5968             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5969             $d->{format} = $defFormat if !$supported;
5970
5971             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5972                                                   $d->{format}, undef, $alloc_size);
5973             print STDERR "new volume ID is '$volid'\n";
5974             $d->{volid} = $volid;
5975             my $path = PVE::Storage::path($cfg, $volid);
5976
5977             PVE::Storage::activate_volumes($cfg,[$volid]);
5978
5979             my $write_zeros = 1;
5980             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5981                 $write_zeros = 0;
5982             }
5983
5984             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5985
5986             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5987             $map->{$virtdev} = $volid;
5988         }
5989
5990         $fh->seek(0, 0) || die "seek failed - $!\n";
5991
5992         my $outfd = new IO::File ($tmpfn, "w") ||
5993             die "unable to write config for VM $vmid\n";
5994
5995         my $cookie = { netcount => 0 };
5996         while (defined(my $line = <$fh>)) {
5997             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5998         }
5999
6000         $fh->close();
6001         $outfd->close();
6002     };
6003
6004     eval {
6005         # enable interrupts
6006         local $SIG{INT} =
6007             local $SIG{TERM} =
6008             local $SIG{QUIT} =
6009             local $SIG{HUP} =
6010             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6011         local $SIG{ALRM} = sub { die "got timeout\n"; };
6012
6013         $oldtimeout = alarm($timeout);
6014
6015         my $parser = sub {
6016             my $line = shift;
6017
6018             print "$line\n";
6019
6020             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6021                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6022                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6023             } elsif ($line =~ m/^CTIME: /) {
6024                 # we correctly received the vma config, so we can disable
6025                 # the timeout now for disk allocation (set to 10 minutes, so
6026                 # that we always timeout if something goes wrong)
6027                 alarm(600);
6028                 &$print_devmap();
6029                 print $fifofh "done\n";
6030                 my $tmp = $oldtimeout || 0;
6031                 $oldtimeout = undef;
6032                 alarm($tmp);
6033                 close($fifofh);
6034             }
6035         };
6036
6037         print "restore vma archive: $dbg_cmdstring\n";
6038         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6039     };
6040     my $err = $@;
6041
6042     alarm($oldtimeout) if $oldtimeout;
6043
6044     my $vollist = [];
6045     foreach my $devname (keys %$devinfo) {
6046         my $volid = $devinfo->{$devname}->{volid};
6047         push @$vollist, $volid if $volid;
6048     }
6049
6050     PVE::Storage::deactivate_volumes($cfg, $vollist);
6051
6052     unlink $mapfifo;
6053
6054     if ($err) {
6055         rmtree $tmpdir;
6056         unlink $tmpfn;
6057
6058         foreach my $devname (keys %$devinfo) {
6059             my $volid = $devinfo->{$devname}->{volid};
6060             next if !$volid;
6061             eval {
6062                 if ($volid =~ m|^/|) {
6063                     unlink $volid || die 'unlink failed\n';
6064                 } else {
6065                     PVE::Storage::vdisk_free($cfg, $volid);
6066                 }
6067                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6068             };
6069             print STDERR "unable to cleanup '$volid' - $@" if $@;
6070         }
6071         die $err;
6072     }
6073
6074     rmtree $tmpdir;
6075
6076     rename($tmpfn, $conffile) ||
6077         die "unable to commit configuration file '$conffile'\n";
6078
6079     PVE::Cluster::cfs_update(); # make sure we read new file
6080
6081     eval { rescan($vmid, 1); };
6082     warn $@ if $@;
6083 }
6084
6085 sub restore_tar_archive {
6086     my ($archive, $vmid, $user, $opts) = @_;
6087
6088     if ($archive ne '-') {
6089         my $firstfile = tar_archive_read_firstfile($archive);
6090         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6091             if $firstfile ne 'qemu-server.conf';
6092     }
6093
6094     my $storecfg = PVE::Storage::config();
6095
6096     # destroy existing data - keep empty config
6097     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6098     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6099
6100     my $tocmd = "/usr/lib/qemu-server/qmextract";
6101
6102     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6103     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6104     $tocmd .= ' --prealloc' if $opts->{prealloc};
6105     $tocmd .= ' --info' if $opts->{info};
6106
6107     # tar option "xf" does not autodetect compression when read from STDIN,
6108     # so we pipe to zcat
6109     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6110         PVE::Tools::shellquote("--to-command=$tocmd");
6111
6112     my $tmpdir = "/var/tmp/vzdumptmp$$";
6113     mkpath $tmpdir;
6114
6115     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6116     local $ENV{VZDUMP_VMID} = $vmid;
6117     local $ENV{VZDUMP_USER} = $user;
6118
6119     my $conffile = PVE::QemuConfig->config_file($vmid);
6120     my $tmpfn = "$conffile.$$.tmp";
6121
6122     # disable interrupts (always do cleanups)
6123     local $SIG{INT} =
6124         local $SIG{TERM} =
6125         local $SIG{QUIT} =
6126         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6127
6128     eval {
6129         # enable interrupts
6130         local $SIG{INT} =
6131             local $SIG{TERM} =
6132             local $SIG{QUIT} =
6133             local $SIG{HUP} =
6134             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6135
6136         if ($archive eq '-') {
6137             print "extracting archive from STDIN\n";
6138             run_command($cmd, input => "<&STDIN");
6139         } else {
6140             print "extracting archive '$archive'\n";
6141             run_command($cmd);
6142         }
6143
6144         return if $opts->{info};
6145
6146         # read new mapping
6147         my $map = {};
6148         my $statfile = "$tmpdir/qmrestore.stat";
6149         if (my $fd = IO::File->new($statfile, "r")) {
6150             while (defined (my $line = <$fd>)) {
6151                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6152                     $map->{$1} = $2 if $1;
6153                 } else {
6154                     print STDERR "unable to parse line in statfile - $line\n";
6155                 }
6156             }
6157             $fd->close();
6158         }
6159
6160         my $confsrc = "$tmpdir/qemu-server.conf";
6161
6162         my $srcfd = new IO::File($confsrc, "r") ||
6163             die "unable to open file '$confsrc'\n";
6164
6165         my $outfd = new IO::File ($tmpfn, "w") ||
6166             die "unable to write config for VM $vmid\n";
6167
6168         my $cookie = { netcount => 0 };
6169         while (defined (my $line = <$srcfd>)) {
6170             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6171         }
6172
6173         $srcfd->close();
6174         $outfd->close();
6175     };
6176     my $err = $@;
6177
6178     if ($err) {
6179
6180         unlink $tmpfn;
6181
6182         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6183
6184         die $err;
6185     }
6186
6187     rmtree $tmpdir;
6188
6189     rename $tmpfn, $conffile ||
6190         die "unable to commit configuration file '$conffile'\n";
6191
6192     PVE::Cluster::cfs_update(); # make sure we read new file
6193
6194     eval { rescan($vmid, 1); };
6195     warn $@ if $@;
6196 };
6197
6198 sub foreach_storage_used_by_vm {
6199     my ($conf, $func) = @_;
6200
6201     my $sidhash = {};
6202
6203     foreach_drive($conf, sub {
6204         my ($ds, $drive) = @_;
6205         return if drive_is_cdrom($drive);
6206
6207         my $volid = $drive->{file};
6208
6209         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6210         $sidhash->{$sid} = $sid if $sid;
6211     });
6212
6213     foreach my $sid (sort keys %$sidhash) {
6214         &$func($sid);
6215     }
6216 }
6217
6218 sub do_snapshots_with_qemu {
6219     my ($storecfg, $volid) = @_;
6220
6221     my $storage_name = PVE::Storage::parse_volume_id($volid);
6222
6223     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6224         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6225         return 1;
6226     }
6227
6228     if ($volid =~ m/\.(qcow2|qed)$/){
6229         return 1;
6230     }
6231
6232     return undef;
6233 }
6234
6235 sub qga_check_running {
6236     my ($vmid, $nowarn) = @_;
6237
6238     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6239     if ($@) {
6240         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6241         return 0;
6242     }
6243     return 1;
6244 }
6245
6246 sub template_create {
6247     my ($vmid, $conf, $disk) = @_;
6248
6249     my $storecfg = PVE::Storage::config();
6250
6251     foreach_drive($conf, sub {
6252         my ($ds, $drive) = @_;
6253
6254         return if drive_is_cdrom($drive);
6255         return if $disk && $ds ne $disk;
6256
6257         my $volid = $drive->{file};
6258         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6259
6260         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6261         $drive->{file} = $voliddst;
6262         $conf->{$ds} = print_drive($vmid, $drive);
6263         PVE::QemuConfig->write_config($vmid, $conf);
6264     });
6265 }
6266
6267 sub qemu_img_convert {
6268     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6269
6270     my $storecfg = PVE::Storage::config();
6271     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6272     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6273
6274     if ($src_storeid && $dst_storeid) {
6275
6276         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6277
6278         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6279         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6280
6281         my $src_format = qemu_img_format($src_scfg, $src_volname);
6282         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6283
6284         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6285         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6286
6287         my $cmd = [];
6288         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6289         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6290         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6291         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6292         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6293         if ($is_zero_initialized) {
6294             push @$cmd, "zeroinit:$dst_path";
6295         } else {
6296             push @$cmd, $dst_path;
6297         }
6298
6299         my $parser = sub {
6300             my $line = shift;
6301             if($line =~ m/\((\S+)\/100\%\)/){
6302                 my $percent = $1;
6303                 my $transferred = int($size * $percent / 100);
6304                 my $remaining = $size - $transferred;
6305
6306                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6307             }
6308
6309         };
6310
6311         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6312         my $err = $@;
6313         die "copy failed: $err" if $err;
6314     }
6315 }
6316
6317 sub qemu_img_format {
6318     my ($scfg, $volname) = @_;
6319
6320     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6321         return $1;
6322     } else {
6323         return "raw";
6324     }
6325 }
6326
6327 sub qemu_drive_mirror {
6328     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6329
6330     $jobs = {} if !$jobs;
6331
6332     my $qemu_target;
6333     my $format;
6334     $jobs->{"drive-$drive"} = {};
6335
6336     if ($dst_volid =~ /^nbd:/) {
6337         $qemu_target = $dst_volid;
6338         $format = "nbd";
6339     } else {
6340         my $storecfg = PVE::Storage::config();
6341         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6342
6343         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6344
6345         $format = qemu_img_format($dst_scfg, $dst_volname);
6346
6347         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6348
6349         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6350     }
6351
6352     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6353     $opts->{format} = $format if $format;
6354
6355     print "drive mirror is starting for drive-$drive\n";
6356
6357     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6358
6359     if (my $err = $@) {
6360         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6361         die "mirroring error: $err";
6362     }
6363
6364     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6365 }
6366
6367 sub qemu_drive_mirror_monitor {
6368     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6369
6370     eval {
6371         my $err_complete = 0;
6372
6373         while (1) {
6374             die "storage migration timed out\n" if $err_complete > 300;
6375
6376             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6377
6378             my $running_mirror_jobs = {};
6379             foreach my $stat (@$stats) {
6380                 next if $stat->{type} ne 'mirror';
6381                 $running_mirror_jobs->{$stat->{device}} = $stat;
6382             }
6383
6384             my $readycounter = 0;
6385
6386             foreach my $job (keys %$jobs) {
6387
6388                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6389                     print "$job : finished\n";
6390                     delete $jobs->{$job};
6391                     next;
6392                 }
6393
6394                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6395
6396                 my $busy = $running_mirror_jobs->{$job}->{busy};
6397                 my $ready = $running_mirror_jobs->{$job}->{ready};
6398                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6399                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6400                     my $remaining = $total - $transferred;
6401                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6402
6403                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6404                 }
6405
6406                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6407             }
6408
6409             last if scalar(keys %$jobs) == 0;
6410
6411             if ($readycounter == scalar(keys %$jobs)) {
6412                 print "all mirroring jobs are ready \n";
6413                 last if $skipcomplete; #do the complete later
6414
6415                 if ($vmiddst && $vmiddst != $vmid) {
6416                     my $agent_running = $qga && qga_check_running($vmid);
6417                     if ($agent_running) {
6418                         print "freeze filesystem\n";
6419                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6420                     } else {
6421                         print "suspend vm\n";
6422                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6423                     }
6424
6425                     # if we clone a disk for a new target vm, we don't switch the disk
6426                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6427
6428                     if ($agent_running) {
6429                         print "unfreeze filesystem\n";
6430                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6431                     } else {
6432                         print "resume vm\n";
6433                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6434                     }
6435
6436                     last;
6437                 } else {
6438
6439                     foreach my $job (keys %$jobs) {
6440                         # try to switch the disk if source and destination are on the same guest
6441                         print "$job: Completing block job...\n";
6442
6443                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6444                         if ($@ =~ m/cannot be completed/) {
6445                             print "$job: Block job cannot be completed, try again.\n";
6446                             $err_complete++;
6447                         }else {
6448                             print "$job: Completed successfully.\n";
6449                             $jobs->{$job}->{complete} = 1;
6450                         }
6451                     }
6452                 }
6453             }
6454             sleep 1;
6455         }
6456     };
6457     my $err = $@;
6458
6459     if ($err) {
6460         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6461         die "mirroring error: $err";
6462     }
6463
6464 }
6465
6466 sub qemu_blockjobs_cancel {
6467     my ($vmid, $jobs) = @_;
6468
6469     foreach my $job (keys %$jobs) {
6470         print "$job: Cancelling block job\n";
6471         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6472         $jobs->{$job}->{cancel} = 1;
6473     }
6474
6475     while (1) {
6476         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6477
6478         my $running_jobs = {};
6479         foreach my $stat (@$stats) {
6480             $running_jobs->{$stat->{device}} = $stat;
6481         }
6482
6483         foreach my $job (keys %$jobs) {
6484
6485             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6486                 print "$job: Done.\n";
6487                 delete $jobs->{$job};
6488             }
6489         }
6490
6491         last if scalar(keys %$jobs) == 0;
6492
6493         sleep 1;
6494     }
6495 }
6496
6497 sub clone_disk {
6498     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6499         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6500
6501     my $newvolid;
6502
6503     if (!$full) {
6504         print "create linked clone of drive $drivename ($drive->{file})\n";
6505         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6506         push @$newvollist, $newvolid;
6507     } else {
6508
6509         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6510         $storeid = $storage if $storage;
6511
6512         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6513         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6514
6515         print "create full clone of drive $drivename ($drive->{file})\n";
6516         my $name = undef;
6517         if (drive_is_cloudinit($drive)) {
6518             $name = "vm-$newvmid-cloudinit";
6519             # cloudinit only supports raw and qcow2 atm:
6520             if ($dst_format eq 'qcow2') {
6521                 $name .= '.qcow2';
6522             } elsif ($dst_format ne 'raw') {
6523                 die "clone: unhandled format for cloudinit image\n";
6524             }
6525         }
6526         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6527         push @$newvollist, $newvolid;
6528
6529         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6530
6531         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6532         if (!$running || $snapname) {
6533             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6534         } else {
6535
6536             my $kvmver = get_running_qemu_version ($vmid);
6537             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6538                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6539                     if $drive->{iothread};
6540             }
6541
6542             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6543         }
6544     }
6545
6546     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6547
6548     my $disk = $drive;
6549     $disk->{format} = undef;
6550     $disk->{file} = $newvolid;
6551     $disk->{size} = $size;
6552
6553     return $disk;
6554 }
6555
6556 # this only works if VM is running
6557 sub get_current_qemu_machine {
6558     my ($vmid) = @_;
6559
6560     my $cmd = { execute => 'query-machines', arguments => {} };
6561     my $res = vm_qmp_command($vmid, $cmd);
6562
6563     my ($current, $default);
6564     foreach my $e (@$res) {
6565         $default = $e->{name} if $e->{'is-default'};
6566         $current = $e->{name} if $e->{'is-current'};
6567     }
6568
6569     # fallback to the default machine if current is not supported by qemu
6570     return $current || $default || 'pc';
6571 }
6572
6573 sub get_running_qemu_version {
6574     my ($vmid) = @_;
6575     my $cmd = { execute => 'query-version', arguments => {} };
6576     my $res = vm_qmp_command($vmid, $cmd);
6577     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6578 }
6579
6580 sub qemu_machine_feature_enabled {
6581     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6582
6583     my $current_major;
6584     my $current_minor;
6585
6586     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6587
6588         $current_major = $3;
6589         $current_minor = $4;
6590
6591     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6592
6593         $current_major = $1;
6594         $current_minor = $2;
6595     }
6596
6597     return 1 if $current_major > $version_major ||
6598                 ($current_major == $version_major &&
6599                  $current_minor >= $version_minor);
6600 }
6601
6602 sub qemu_machine_pxe {
6603     my ($vmid, $conf, $machine) = @_;
6604
6605     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6606
6607     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6608         $machine .= '.pxe';
6609     }
6610
6611     return $machine;
6612 }
6613
6614 sub qemu_use_old_bios_files {
6615     my ($machine_type) = @_;
6616
6617     return if !$machine_type;
6618
6619     my $use_old_bios_files = undef;
6620
6621     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6622         $machine_type = $1;
6623         $use_old_bios_files = 1;
6624     } else {
6625         my $kvmver = kvm_user_version();
6626         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6627         # load new efi bios files on migration. So this hack is required to allow
6628         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6629         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6630         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6631     }
6632
6633     return ($use_old_bios_files, $machine_type);
6634 }
6635
6636 sub create_efidisk {
6637     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6638
6639     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6640
6641     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6642     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6643     PVE::Storage::activate_volumes($storecfg, [$volid]);
6644
6645     my $path = PVE::Storage::path($storecfg, $volid);
6646     eval {
6647         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6648     };
6649     die "Copying EFI vars image failed: $@" if $@;
6650
6651     return ($volid, $vars_size);
6652 }
6653
6654 sub lspci {
6655
6656     my $devices = {};
6657
6658     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6659             my (undef, $id, $function) = @_;
6660             my $res = { id => $id, function => $function};
6661             push @{$devices->{$id}}, $res;
6662     });
6663
6664     # Entries should be sorted by functions.
6665     foreach my $id (keys %$devices) {
6666         my $dev = $devices->{$id};
6667         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6668     }
6669
6670     return $devices;
6671 }
6672
6673 sub vm_iothreads_list {
6674     my ($vmid) = @_;
6675
6676     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6677
6678     my $iothreads = {};
6679     foreach my $iothread (@$res) {
6680         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6681     }
6682
6683     return $iothreads;
6684 }
6685
6686 sub scsihw_infos {
6687     my ($conf, $drive) = @_;
6688
6689     my $maxdev = 0;
6690
6691     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6692         $maxdev = 7;
6693     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6694         $maxdev = 1;
6695     } else {
6696         $maxdev = 256;
6697     }
6698
6699     my $controller = int($drive->{index} / $maxdev);
6700     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6701
6702     return ($maxdev, $controller, $controller_prefix);
6703 }
6704
6705 sub add_hyperv_enlightenments {
6706     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6707
6708     return if $winversion < 6;
6709     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6710
6711     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6712
6713     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6714         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6715         push @$cpuFlags , 'hv_vapic';
6716         push @$cpuFlags , 'hv_time';
6717     } else {
6718         push @$cpuFlags , 'hv_spinlocks=0xffff';
6719     }
6720
6721     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6722         push @$cpuFlags , 'hv_reset';
6723         push @$cpuFlags , 'hv_vpindex';
6724         push @$cpuFlags , 'hv_runtime';
6725     }
6726
6727     if ($winversion >= 7) {
6728         push @$cpuFlags , 'hv_relaxed';
6729
6730         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
6731             push @$cpuFlags , 'hv_synic';
6732             push @$cpuFlags , 'hv_stimer';
6733         }
6734     }
6735 }
6736
6737 sub windows_version {
6738     my ($ostype) = @_;
6739
6740     return 0 if !$ostype;
6741
6742     my $winversion = 0;
6743
6744     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6745         $winversion = 5;
6746     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6747         $winversion = 6;
6748     } elsif ($ostype =~ m/^win(\d+)$/) {
6749         $winversion = $1;
6750     }
6751
6752     return $winversion;
6753 }
6754
6755 sub resolve_dst_disk_format {
6756         my ($storecfg, $storeid, $src_volname, $format) = @_;
6757         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6758
6759         if (!$format) {
6760             # if no target format is specified, use the source disk format as hint
6761             if ($src_volname) {
6762                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6763                 $format = qemu_img_format($scfg, $src_volname);
6764             } else {
6765                 return $defFormat;
6766             }
6767         }
6768
6769         # test if requested format is supported - else use default
6770         my $supported = grep { $_ eq $format } @$validFormats;
6771         $format = $defFormat if !$supported;
6772         return $format;
6773 }
6774
6775 sub resolve_first_disk {
6776     my $conf = shift;
6777     my @disks = PVE::QemuServer::valid_drive_names();
6778     my $firstdisk;
6779     foreach my $ds (reverse @disks) {
6780         next if !$conf->{$ds};
6781         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6782         next if PVE::QemuServer::drive_is_cdrom($disk);
6783         $firstdisk = $ds;
6784     }
6785     return $firstdisk;
6786 }
6787
6788 sub generate_uuid {
6789     my ($uuid, $uuid_str);
6790     UUID::generate($uuid);
6791     UUID::unparse($uuid, $uuid_str);
6792     return $uuid_str;
6793 }
6794
6795 sub generate_smbios1_uuid {
6796     return "uuid=".generate_uuid();
6797 }
6798
6799 # bash completion helper
6800
6801 sub complete_backup_archives {
6802     my ($cmdname, $pname, $cvalue) = @_;
6803
6804     my $cfg = PVE::Storage::config();
6805
6806     my $storeid;
6807
6808     if ($cvalue =~ m/^([^:]+):/) {
6809         $storeid = $1;
6810     }
6811
6812     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6813
6814     my $res = [];
6815     foreach my $id (keys %$data) {
6816         foreach my $item (@{$data->{$id}}) {
6817             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6818             push @$res, $item->{volid} if defined($item->{volid});
6819         }
6820     }
6821
6822     return $res;
6823 }
6824
6825 my $complete_vmid_full = sub {
6826     my ($running) = @_;
6827
6828     my $idlist = vmstatus();
6829
6830     my $res = [];
6831
6832     foreach my $id (keys %$idlist) {
6833         my $d = $idlist->{$id};
6834         if (defined($running)) {
6835             next if $d->{template};
6836             next if $running && $d->{status} ne 'running';
6837             next if !$running && $d->{status} eq 'running';
6838         }
6839         push @$res, $id;
6840
6841     }
6842     return $res;
6843 };
6844
6845 sub complete_vmid {
6846     return &$complete_vmid_full();
6847 }
6848
6849 sub complete_vmid_stopped {
6850     return &$complete_vmid_full(0);
6851 }
6852
6853 sub complete_vmid_running {
6854     return &$complete_vmid_full(1);
6855 }
6856
6857 sub complete_storage {
6858
6859     my $cfg = PVE::Storage::config();
6860     my $ids = $cfg->{ids};
6861
6862     my $res = [];
6863     foreach my $sid (keys %$ids) {
6864         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6865         next if !$ids->{$sid}->{content}->{images};
6866         push @$res, $sid;
6867     }
6868
6869     return $res;
6870 }
6871
6872 sub nbd_stop {
6873     my ($vmid) = @_;
6874
6875     vm_mon_cmd($vmid, 'nbd-server-stop');
6876 }
6877
6878 1;