PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 #no warnings 'redefine';
  84
  85 sub cgroups_write {
  86    my ($controller, $vmid, $option, $value) = @_;
  87
  88    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  89    PVE::ProcFSTools::write_proc_entry($path, $value);
  90
  91 }
  92
  93 my $nodename = PVE::INotify::nodename();
  94
  95 mkdir "/etc/pve/nodes/$nodename";
  96 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  97 mkdir $confdir;
  98
  99 my $var_run_tmpdir = "/var/run/qemu-server";
 100 mkdir $var_run_tmpdir;
 101
 102 my $lock_dir = "/var/lock/qemu-server";
 103 mkdir $lock_dir;
 104
 105 my $pcisysfs = "/sys/bus/pci";
 106
 107 my $cpu_vendor_list = {
 108     # Intel CPUs
 109     486 => 'GenuineIntel',
 110     pentium => 'GenuineIntel',
 111     pentium2  => 'GenuineIntel',
 112     pentium3  => 'GenuineIntel',
 113     coreduo => 'GenuineIntel',
 114     core2duo => 'GenuineIntel',
 115     Conroe  => 'GenuineIntel',
 116     Penryn  => 'GenuineIntel',
 117     Nehalem  => 'GenuineIntel',
 118     'Nehalem-IBRS'  => 'GenuineIntel',
 119     Westmere => 'GenuineIntel',
 120     'Westmere-IBRS' => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     'SandyBridge-IBRS' => 'GenuineIntel',
 123     IvyBridge => 'GenuineIntel',
 124     'IvyBridge-IBRS' => 'GenuineIntel',
 125     Haswell => 'GenuineIntel',
 126     'Haswell-IBRS' => 'GenuineIntel',
 127     'Haswell-noTSX' => 'GenuineIntel',
 128     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 129     Broadwell => 'GenuineIntel',
 130     'Broadwell-IBRS' => 'GenuineIntel',
 131     'Broadwell-noTSX' => 'GenuineIntel',
 132     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 133     'Skylake-Client' => 'GenuineIntel',
 134     'Skylake-Client-IBRS' => 'GenuineIntel',
 135     'Skylake-Server' => 'GenuineIntel',
 136     'Skylake-Server-IBRS' => 'GenuineIntel',
 137
 138     # AMD CPUs
 139     athlon => 'AuthenticAMD',
 140     phenom  => 'AuthenticAMD',
 141     Opteron_G1  => 'AuthenticAMD',
 142     Opteron_G2  => 'AuthenticAMD',
 143     Opteron_G3  => 'AuthenticAMD',
 144     Opteron_G4  => 'AuthenticAMD',
 145     Opteron_G5  => 'AuthenticAMD',
 146     EPYC => 'AuthenticAMD',
 147     'EPYC-IBPB' => 'AuthenticAMD',
 148
 149     # generic types, use vendor from host node
 150     host => 'default',
 151     kvm32 => 'default',
 152     kvm64 => 'default',
 153     qemu32 => 'default',
 154     qemu64 => 'default',
 155     max => 'default',
 156 };
 157
 158 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 159
 160 my $cpu_fmt = {
 161     cputype => {
 162         description => "Emulated CPU type.",
 163         type => 'string',
 164         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 165         default => 'kvm64',
 166         default_key => 1,
 167     },
 168     hidden => {
 169         description => "Do not identify as a KVM virtual machine.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 0
 173     },
 174     flags => {
 175         description => "List of additional CPU flags separated by ';'."
 176                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 177                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 178         format_description => '+FLAG[;-FLAG...]',
 179         type => 'string',
 180         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 181         optional => 1,
 182     },
 183 };
 184
 185 my $watchdog_fmt = {
 186     model => {
 187         default_key => 1,
 188         type => 'string',
 189         enum => [qw(i6300esb ib700)],
 190         description => "Watchdog type to emulate.",
 191         default => 'i6300esb',
 192         optional => 1,
 193     },
 194     action => {
 195         type => 'string',
 196         enum => [qw(reset shutdown poweroff pause debug none)],
 197         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 198         optional => 1,
 199     },
 200 };
 201 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 202
 203 my $agent_fmt = {
 204     enabled => {
 205         description => "Enable/disable Qemu GuestAgent.",
 206         type => 'boolean',
 207         default => 0,
 208         default_key => 1,
 209     },
 210     fstrim_cloned_disks => {
 211         description => "Run fstrim after cloning/moving a disk.",
 212         type => 'boolean',
 213         optional => 1,
 214         default => 0
 215     },
 216 };
 217
 218 my $confdesc = {
 219     onboot => {
 220         optional => 1,
 221         type => 'boolean',
 222         description => "Specifies whether a VM will be started during system bootup.",
 223         default => 0,
 224     },
 225     autostart => {
 226         optional => 1,
 227         type => 'boolean',
 228         description => "Automatic restart after crash (currently ignored).",
 229         default => 0,
 230     },
 231     hotplug => {
 232         optional => 1,
 233         type => 'string', format => 'pve-hotplug-features',
 234         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 235         default => 'network,disk,usb',
 236     },
 237     reboot => {
 238         optional => 1,
 239         type => 'boolean',
 240         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 241         default => 1,
 242     },
 243     lock => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Lock/unlock the VM.",
 247         enum => [qw(migrate backup snapshot rollback)],
 248     },
 249     cpulimit => {
 250         optional => 1,
 251         type => 'number',
 252         description => "Limit of CPU usage.",
 253         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 254         minimum => 0,
 255         maximum => 128,
 256         default => 0,
 257     },
 258     cpuunits => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "CPU weight for a VM.",
 262         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 263         minimum => 2,
 264         maximum => 262144,
 265         default => 1024,
 266     },
 267     memory => {
 268         optional => 1,
 269         type => 'integer',
 270         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 271         minimum => 16,
 272         default => 512,
 273     },
 274     balloon => {
 275         optional => 1,
 276         type => 'integer',
 277         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 278         minimum => 0,
 279     },
 280     shares => {
 281         optional => 1,
 282         type => 'integer',
 283         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 284         minimum => 0,
 285         maximum => 50000,
 286         default => 1000,
 287     },
 288     keyboard => {
 289         optional => 1,
 290         type => 'string',
 291         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 292                        "It should not be necessary to set it.",
 293         enum => PVE::Tools::kvmkeymaplist(),
 294         default => undef,
 295     },
 296     name => {
 297         optional => 1,
 298         type => 'string', format => 'dns-name',
 299         description => "Set a name for the VM. Only used on the configuration web interface.",
 300     },
 301     scsihw => {
 302         optional => 1,
 303         type => 'string',
 304         description => "SCSI controller model",
 305         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 306         default => 'lsi',
 307     },
 308     description => {
 309         optional => 1,
 310         type => 'string',
 311         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 312     },
 313     ostype => {
 314         optional => 1,
 315         type => 'string',
 316         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 317         description => "Specify guest operating system.",
 318         verbose_description => <<EODESC,
 319 Specify guest operating system. This is used to enable special
 320 optimization/features for specific operating systems:
 321
 322 [horizontal]
 323 other;; unspecified OS
 324 wxp;; Microsoft Windows XP
 325 w2k;; Microsoft Windows 2000
 326 w2k3;; Microsoft Windows 2003
 327 w2k8;; Microsoft Windows 2008
 328 wvista;; Microsoft Windows Vista
 329 win7;; Microsoft Windows 7
 330 win8;; Microsoft Windows 8/2012/2012r2
 331 win10;; Microsoft Windows 10/2016
 332 l24;; Linux 2.4 Kernel
 333 l26;; Linux 2.6/3.X Kernel
 334 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 335 EODESC
 336     },
 337     boot => {
 338         optional => 1,
 339         type => 'string',
 340         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 341         pattern => '[acdn]{1,4}',
 342         default => 'cdn',
 343     },
 344     bootdisk => {
 345         optional => 1,
 346         type => 'string', format => 'pve-qm-bootdisk',
 347         description => "Enable booting from specified disk.",
 348         pattern => '(ide|sata|scsi|virtio)\d+',
 349     },
 350     smp => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "The number of CPUs. Please use option -sockets instead.",
 354         minimum => 1,
 355         default => 1,
 356     },
 357     sockets => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "The number of CPU sockets.",
 361         minimum => 1,
 362         default => 1,
 363     },
 364     cores => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "The number of cores per socket.",
 368         minimum => 1,
 369         default => 1,
 370     },
 371     numa => {
 372         optional => 1,
 373         type => 'boolean',
 374         description => "Enable/disable NUMA.",
 375         default => 0,
 376     },
 377     hugepages => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Enable/disable hugepages memory.",
 381         enum => [qw(any 2 1024)],
 382     },
 383     vcpus => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "Number of hotplugged vcpus.",
 387         minimum => 1,
 388         default => 0,
 389     },
 390     acpi => {
 391         optional => 1,
 392         type => 'boolean',
 393         description => "Enable/disable ACPI.",
 394         default => 1,
 395     },
 396     agent => {
 397         optional => 1,
 398         description => "Enable/disable Qemu GuestAgent and its properties.",
 399         type => 'string',
 400         format => $agent_fmt,
 401     },
 402     kvm => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Enable/disable KVM hardware virtualization.",
 406         default => 1,
 407     },
 408     tdf => {
 409         optional => 1,
 410         type => 'boolean',
 411         description => "Enable/disable time drift fix.",
 412         default => 0,
 413     },
 414     localtime => {
 415         optional => 1,
 416         type => 'boolean',
 417         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 418     },
 419     freeze => {
 420         optional => 1,
 421         type => 'boolean',
 422         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 423     },
 424     vga => {
 425         optional => 1,
 426         type => 'string',
 427         description => "Select the VGA type.",
 428         verbose_description => "Select the VGA type. If you want to use high resolution" .
 429             " modes (>= 1280x1024x16) then you should use the options " .
 430             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 431             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 432             "display sever. For win* OS you can select how many independent " .
 433             "displays you want, Linux guests can add displays them self. " .
 434             "You can also run without any graphic card, using a serial device" .
 435             " as terminal.",
 436         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 437     },
 438     watchdog => {
 439         optional => 1,
 440         type => 'string', format => 'pve-qm-watchdog',
 441         description => "Create a virtual hardware watchdog device.",
 442         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 443             " (by a guest action), the watchdog must be periodically polled " .
 444             "by an agent inside the guest or else the watchdog will reset " .
 445             "the guest (or execute the respective action specified)",
 446     },
 447     startdate => {
 448         optional => 1,
 449         type => 'string',
 450         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 451         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 452         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 453         default => 'now',
 454     },
 455     startup =>  get_standard_option('pve-startup-order'),
 456     template => {
 457         optional => 1,
 458         type => 'boolean',
 459         description => "Enable/disable Template.",
 460         default => 0,
 461     },
 462     args => {
 463         optional => 1,
 464         type => 'string',
 465         description => "Arbitrary arguments passed to kvm.",
 466         verbose_description => <<EODESCR,
 467 Arbitrary arguments passed to kvm, for example:
 468
 469 args: -no-reboot -no-hpet
 470
 471 NOTE: this option is for experts only.
 472 EODESCR
 473     },
 474     tablet => {
 475         optional => 1,
 476         type => 'boolean',
 477         default => 1,
 478         description => "Enable/disable the USB tablet device.",
 479         verbose_description => "Enable/disable the USB tablet device. This device is " .
 480             "usually needed to allow absolute mouse positioning with VNC. " .
 481             "Else the mouse runs out of sync with normal VNC clients. " .
 482             "If you're running lots of console-only guests on one host, " .
 483             "you may consider disabling this to save some context switches. " .
 484             "This is turned off by default if you use spice (-vga=qxl).",
 485     },
 486     migrate_speed => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 490         minimum => 0,
 491         default => 0,
 492     },
 493     migrate_downtime => {
 494         optional => 1,
 495         type => 'number',
 496         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 497         minimum => 0,
 498         default => 0.1,
 499     },
 500     cdrom => {
 501         optional => 1,
 502         type => 'string', format => 'pve-qm-ide',
 503         typetext => '<volume>',
 504         description => "This is an alias for option -ide2",
 505     },
 506     cpu => {
 507         optional => 1,
 508         description => "Emulated CPU type.",
 509         type => 'string',
 510         format => $cpu_fmt,
 511     },
 512     parent => get_standard_option('pve-snapshot-name', {
 513         optional => 1,
 514         description => "Parent snapshot name. This is used internally, and should not be modified.",
 515     }),
 516     snaptime => {
 517         optional => 1,
 518         description => "Timestamp for snapshots.",
 519         type => 'integer',
 520         minimum => 0,
 521     },
 522     vmstate => {
 523         optional => 1,
 524         type => 'string', format => 'pve-volume-id',
 525         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 526     },
 527     vmstatestorage => get_standard_option('pve-storage-id', {
 528         description => "Default storage for VM state volumes/files.",
 529         optional => 1,
 530     }),
 531     machine => {
 532         description => "Specific the Qemu machine type.",
 533         type => 'string',
 534         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 535         maxLength => 40,
 536         optional => 1,
 537     },
 538     smbios1 => {
 539         description => "Specify SMBIOS type 1 fields.",
 540         type => 'string', format => 'pve-qm-smbios1',
 541         maxLength => 256,
 542         optional => 1,
 543     },
 544     protection => {
 545         optional => 1,
 546         type => 'boolean',
 547         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 548         default => 0,
 549     },
 550     bios => {
 551         optional => 1,
 552         type => 'string',
 553         enum => [ qw(seabios ovmf) ],
 554         description => "Select BIOS implementation.",
 555         default => 'seabios',
 556     },
 557 };
 558
 559 my $confdesc_cloudinit = {
 560     citype => {
 561         optional => 1,
 562         type => 'string',
 563         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 564         enum => ['configdrive2', 'nocloud'],
 565     },
 566     ciuser => {
 567         optional => 1,
 568         type => 'string',
 569         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 570     },
 571     cipassword => {
 572         optional => 1,
 573         type => 'string',
 574         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 575     },
 576     searchdomain => {
 577         optional => 1,
 578         type => 'string',
 579         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 580     },
 581     nameserver => {
 582         optional => 1,
 583         type => 'string', format => 'address-list',
 584         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 585     },
 586     sshkeys => {
 587         optional => 1,
 588         type => 'string',
 589         format => 'urlencoded',
 590         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 591     },
 592 };
 593
 594 # what about other qemu settings ?
 595 #cpu => 'string',
 596 #machine => 'string',
 597 #fda => 'file',
 598 #fdb => 'file',
 599 #mtdblock => 'file',
 600 #sd => 'file',
 601 #pflash => 'file',
 602 #snapshot => 'bool',
 603 #bootp => 'file',
 604 ##tftp => 'dir',
 605 ##smb => 'dir',
 606 #kernel => 'file',
 607 #append => 'string',
 608 #initrd => 'file',
 609 ##soundhw => 'string',
 610
 611 while (my ($k, $v) = each %$confdesc) {
 612     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 613 }
 614
 615 my $MAX_IDE_DISKS = 4;
 616 my $MAX_SCSI_DISKS = 14;
 617 my $MAX_VIRTIO_DISKS = 16;
 618 my $MAX_SATA_DISKS = 6;
 619 my $MAX_USB_DEVICES = 5;
 620 my $MAX_NETS = 32;
 621 my $MAX_UNUSED_DISKS = 8;
 622 my $MAX_HOSTPCI_DEVICES = 4;
 623 my $MAX_SERIAL_PORTS = 4;
 624 my $MAX_PARALLEL_PORTS = 3;
 625 my $MAX_NUMA = 8;
 626
 627 my $numa_fmt = {
 628     cpus => {
 629         type => "string",
 630         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 631         description => "CPUs accessing this NUMA node.",
 632         format_description => "id[-id];...",
 633     },
 634     memory => {
 635         type => "number",
 636         description => "Amount of memory this NUMA node provides.",
 637         optional => 1,
 638     },
 639     hostnodes => {
 640         type => "string",
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => "Host NUMA nodes to use.",
 643         format_description => "id[-id];...",
 644         optional => 1,
 645     },
 646     policy => {
 647         type => 'string',
 648         enum => [qw(preferred bind interleave)],
 649         description => "NUMA allocation policy.",
 650         optional => 1,
 651     },
 652 };
 653 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 654 my $numadesc = {
 655     optional => 1,
 656     type => 'string', format => $numa_fmt,
 657     description => "NUMA topology.",
 658 };
 659 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 660
 661 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 662     $confdesc->{"numa$i"} = $numadesc;
 663 }
 664
 665 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 666                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 667                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 668 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 669
 670 my $net_fmt_bridge_descr = <<__EOD__;
 671 Bridge to attach the network device to. The Proxmox VE standard bridge
 672 is called 'vmbr0'.
 673
 674 If you do not specify a bridge, we create a kvm user (NATed) network
 675 device, which provides DHCP and DNS services. The following addresses
 676 are used:
 677
 678  10.0.2.2   Gateway
 679  10.0.2.3   DNS Server
 680  10.0.2.4   SMB Server
 681
 682 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 683 __EOD__
 684
 685 my $net_fmt = {
 686     macaddr => {
 687         type => 'string',
 688         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 689         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 690         format_description => "XX:XX:XX:XX:XX:XX",
 691         optional => 1,
 692     },
 693     model => {
 694         type => 'string',
 695         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 696         enum => $nic_model_list,
 697         default_key => 1,
 698     },
 699     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 700     bridge => {
 701         type => 'string',
 702         description => $net_fmt_bridge_descr,
 703         format_description => 'bridge',
 704         optional => 1,
 705     },
 706     queues => {
 707         type => 'integer',
 708         minimum => 0, maximum => 16,
 709         description => 'Number of packet queues to be used on the device.',
 710         optional => 1,
 711     },
 712     rate => {
 713         type => 'number',
 714         minimum => 0,
 715         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 716         optional => 1,
 717     },
 718     tag => {
 719         type => 'integer',
 720         minimum => 1, maximum => 4094,
 721         description => 'VLAN tag to apply to packets on this interface.',
 722         optional => 1,
 723     },
 724     trunks => {
 725         type => 'string',
 726         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 727         description => 'VLAN trunks to pass through this interface.',
 728         format_description => 'vlanid[;vlanid...]',
 729         optional => 1,
 730     },
 731     firewall => {
 732         type => 'boolean',
 733         description => 'Whether this interface should be protected by the firewall.',
 734         optional => 1,
 735     },
 736     link_down => {
 737         type => 'boolean',
 738         description => 'Whether this interface should be disconnected (like pulling the plug).',
 739         optional => 1,
 740     },
 741 };
 742
 743 my $netdesc = {
 744     optional => 1,
 745     type => 'string', format => $net_fmt,
 746     description => "Specify network devices.",
 747 };
 748
 749 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 750
 751 my $ipconfig_fmt = {
 752     ip => {
 753         type => 'string',
 754         format => 'pve-ipv4-config',
 755         format_description => 'IPv4Format/CIDR',
 756         description => 'IPv4 address in CIDR format.',
 757         optional => 1,
 758         default => 'dhcp',
 759     },
 760     gw => {
 761         type => 'string',
 762         format => 'ipv4',
 763         format_description => 'GatewayIPv4',
 764         description => 'Default gateway for IPv4 traffic.',
 765         optional => 1,
 766         requires => 'ip',
 767     },
 768     ip6 => {
 769         type => 'string',
 770         format => 'pve-ipv6-config',
 771         format_description => 'IPv6Format/CIDR',
 772         description => 'IPv6 address in CIDR format.',
 773         optional => 1,
 774         default => 'dhcp',
 775     },
 776     gw6 => {
 777         type => 'string',
 778         format => 'ipv6',
 779         format_description => 'GatewayIPv6',
 780         description => 'Default gateway for IPv6 traffic.',
 781         optional => 1,
 782         requires => 'ip6',
 783     },
 784 };
 785 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 786 my $ipconfigdesc = {
 787     optional => 1,
 788     type => 'string', format => 'pve-qm-ipconfig',
 789     description => <<'EODESCR',
 790 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 791
 792 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 793
 794 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 795 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 796
 797 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 798 EODESCR
 799 };
 800 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 801
 802 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 803     $confdesc->{"net$i"} = $netdesc;
 804     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 805 }
 806
 807 foreach my $key (keys %$confdesc_cloudinit) {
 808     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 809 }
 810
 811 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 812 sub verify_volume_id_or_qm_path {
 813     my ($volid, $noerr) = @_;
 814
 815     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 816         return $volid;
 817     }
 818
 819     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 820     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 821     if ($@) {
 822         return undef if $noerr;
 823         die $@;
 824     }
 825     return $volid;
 826 }
 827
 828 my $drivename_hash;
 829
 830 my %drivedesc_base = (
 831     volume => { alias => 'file' },
 832     file => {
 833         type => 'string',
 834         format => 'pve-volume-id-or-qm-path',
 835         default_key => 1,
 836         format_description => 'volume',
 837         description => "The drive's backing volume.",
 838     },
 839     media => {
 840         type => 'string',
 841         enum => [qw(cdrom disk)],
 842         description => "The drive's media type.",
 843         default => 'disk',
 844         optional => 1
 845     },
 846     cyls => {
 847         type => 'integer',
 848         description => "Force the drive's physical geometry to have a specific cylinder count.",
 849         optional => 1
 850     },
 851     heads => {
 852         type => 'integer',
 853         description => "Force the drive's physical geometry to have a specific head count.",
 854         optional => 1
 855     },
 856     secs => {
 857         type => 'integer',
 858         description => "Force the drive's physical geometry to have a specific sector count.",
 859         optional => 1
 860     },
 861     trans => {
 862         type => 'string',
 863         enum => [qw(none lba auto)],
 864         description => "Force disk geometry bios translation mode.",
 865         optional => 1,
 866     },
 867     snapshot => {
 868         type => 'boolean',
 869         description => "Controls qemu's snapshot mode feature."
 870             . " If activated, changes made to the disk are temporary and will"
 871             . " be discarded when the VM is shutdown.",
 872         optional => 1,
 873     },
 874     cache => {
 875         type => 'string',
 876         enum => [qw(none writethrough writeback unsafe directsync)],
 877         description => "The drive's cache mode",
 878         optional => 1,
 879     },
 880     format => get_standard_option('pve-qm-image-format'),
 881     size => {
 882         type => 'string',
 883         format => 'disk-size',
 884         format_description => 'DiskSize',
 885         description => "Disk size. This is purely informational and has no effect.",
 886         optional => 1,
 887     },
 888     backup => {
 889         type => 'boolean',
 890         description => "Whether the drive should be included when making backups.",
 891         optional => 1,
 892     },
 893     replicate => {
 894         type => 'boolean',
 895         description => 'Whether the drive should considered for replication jobs.',
 896         optional => 1,
 897         default => 1,
 898     },
 899     rerror => {
 900         type => 'string',
 901         enum => [qw(ignore report stop)],
 902         description => 'Read error action.',
 903         optional => 1,
 904     },
 905     werror => {
 906         type => 'string',
 907         enum => [qw(enospc ignore report stop)],
 908         description => 'Write error action.',
 909         optional => 1,
 910     },
 911     aio => {
 912         type => 'string',
 913         enum => [qw(native threads)],
 914         description => 'AIO type to use.',
 915         optional => 1,
 916     },
 917     discard => {
 918         type => 'string',
 919         enum => [qw(ignore on)],
 920         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 921         optional => 1,
 922     },
 923     detect_zeroes => {
 924         type => 'boolean',
 925         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 926         optional => 1,
 927     },
 928     serial => {
 929         type => 'string',
 930         format => 'urlencoded',
 931         format_description => 'serial',
 932         maxLength => 20*3, # *3 since it's %xx url enoded
 933         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 934         optional => 1,
 935     },
 936     shared => {
 937         type => 'boolean',
 938         description => 'Mark this locally-managed volume as available on all nodes',
 939         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 940         optional => 1,
 941         default => 0,
 942     }
 943 );
 944
 945 my %iothread_fmt = ( iothread => {
 946         type => 'boolean',
 947         description => "Whether to use iothreads for this drive",
 948         optional => 1,
 949 });
 950
 951 my %model_fmt = (
 952     model => {
 953         type => 'string',
 954         format => 'urlencoded',
 955         format_description => 'model',
 956         maxLength => 40*3, # *3 since it's %xx url enoded
 957         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 958         optional => 1,
 959     },
 960 );
 961
 962 my %queues_fmt = (
 963     queues => {
 964         type => 'integer',
 965         description => "Number of queues.",
 966         minimum => 2,
 967         optional => 1
 968     }
 969 );
 970
 971 my %scsiblock_fmt = (
 972     scsiblock => {
 973         type => 'boolean',
 974         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 975         optional => 1,
 976         default => 0,
 977     },
 978 );
 979
 980 my $add_throttle_desc = sub {
 981     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 982     my $d = {
 983         type => $type,
 984         format_description => $unit,
 985         description => "Maximum $what in $longunit.",
 986         optional => 1,
 987     };
 988     $d->{minimum} = $minimum if defined($minimum);
 989     $drivedesc_base{$key} = $d;
 990 };
 991 # throughput: (leaky bucket)
 992 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 993 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 994 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 995 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 996 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 997 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 998 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 999 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1000 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1001
1002 # pools: (pool of IO before throttling starts taking effect)
1003 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1004 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1005 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1006 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1007 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1008 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1009
1010 # burst lengths
1011 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1012 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1013 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1014 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1015 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1016 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1017
1018 # legacy support
1019 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1020 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1021 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1022 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1023
1024 my $ide_fmt = {
1025     %drivedesc_base,
1026     %model_fmt,
1027 };
1028 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1029
1030 my $idedesc = {
1031     optional => 1,
1032     type => 'string', format => $ide_fmt,
1033     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1036
1037 my $scsi_fmt = {
1038     %drivedesc_base,
1039     %iothread_fmt,
1040     %queues_fmt,
1041     %scsiblock_fmt,
1042 };
1043 my $scsidesc = {
1044     optional => 1,
1045     type => 'string', format => $scsi_fmt,
1046     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1047 };
1048 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1049
1050 my $sata_fmt = {
1051     %drivedesc_base,
1052 };
1053 my $satadesc = {
1054     optional => 1,
1055     type => 'string', format => $sata_fmt,
1056     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1057 };
1058 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1059
1060 my $virtio_fmt = {
1061     %drivedesc_base,
1062     %iothread_fmt,
1063 };
1064 my $virtiodesc = {
1065     optional => 1,
1066     type => 'string', format => $virtio_fmt,
1067     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1068 };
1069 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1070
1071 my $alldrive_fmt = {
1072     %drivedesc_base,
1073     %iothread_fmt,
1074     %model_fmt,
1075     %queues_fmt,
1076     %scsiblock_fmt,
1077 };
1078
1079 my $efidisk_fmt = {
1080     volume => { alias => 'file' },
1081     file => {
1082         type => 'string',
1083         format => 'pve-volume-id-or-qm-path',
1084         default_key => 1,
1085         format_description => 'volume',
1086         description => "The drive's backing volume.",
1087     },
1088     format => get_standard_option('pve-qm-image-format'),
1089     size => {
1090         type => 'string',
1091         format => 'disk-size',
1092         format_description => 'DiskSize',
1093         description => "Disk size. This is purely informational and has no effect.",
1094         optional => 1,
1095     },
1096 };
1097
1098 my $efidisk_desc = {
1099     optional => 1,
1100     type => 'string', format => $efidisk_fmt,
1101     description => "Configure a Disk for storing EFI vars",
1102 };
1103
1104 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1105
1106 my $usb_fmt = {
1107     host => {
1108         default_key => 1,
1109         type => 'string', format => 'pve-qm-usb-device',
1110         format_description => 'HOSTUSBDEVICE|spice',
1111         description => <<EODESCR,
1112 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1113
1114  'bus-port(.port)*' (decimal numbers) or
1115  'vendor_id:product_id' (hexadeciaml numbers) or
1116  'spice'
1117
1118 You can use the 'lsusb -t' command to list existing usb devices.
1119
1120 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1121
1122 The value 'spice' can be used to add a usb redirection devices for spice.
1123 EODESCR
1124     },
1125     usb3 => {
1126         optional => 1,
1127         type => 'boolean',
1128         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1129         default => 0,
1130     },
1131 };
1132
1133 my $usbdesc = {
1134     optional => 1,
1135     type => 'string', format => $usb_fmt,
1136     description => "Configure an USB device (n is 0 to 4).",
1137 };
1138 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1139
1140 # NOTE: the match-groups of this regex are used in parse_hostpci
1141 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1142 my $hostpci_fmt = {
1143     host => {
1144         default_key => 1,
1145         type => 'string',
1146         pattern => qr/$PCIRE(;$PCIRE)*/,
1147         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1148         description => <<EODESCR,
1149 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1150 of PCI virtual functions of the host. HOSTPCIID syntax is:
1151
1152 'bus:dev.func' (hexadecimal numbers)
1153
1154 You can us the 'lspci' command to list existing PCI devices.
1155 EODESCR
1156     },
1157     rombar => {
1158         type => 'boolean',
1159         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1160         optional => 1,
1161         default => 1,
1162     },
1163     romfile => {
1164         type => 'string',
1165         pattern => '[^,;]+',
1166         format_description => 'string',
1167         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1168         optional => 1,
1169     },
1170     pcie => {
1171         type => 'boolean',
1172         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1173         optional => 1,
1174         default => 0,
1175     },
1176     'x-vga' => {
1177         type => 'boolean',
1178         description =>  "Enable vfio-vga device support.",
1179         optional => 1,
1180         default => 0,
1181     },
1182 };
1183 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1184
1185 my $hostpcidesc = {
1186         optional => 1,
1187         type => 'string', format => 'pve-qm-hostpci',
1188         description => "Map host PCI devices into guest.",
1189         verbose_description =>  <<EODESCR,
1190 Map host PCI devices into guest.
1191
1192 NOTE: This option allows direct access to host hardware. So it is no longer
1193 possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1199
1200 my $serialdesc = {
1201         optional => 1,
1202         type => 'string',
1203         pattern => '(/dev/.+|socket)',
1204         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1205         verbose_description =>  <<EODESCR,
1206 Create a serial device inside the VM (n is 0 to 3), and pass through a
1207 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1208 host side (use 'qm terminal' to open a terminal connection).
1209
1210 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1211
1212 CAUTION: Experimental! User reported problems with this option.
1213 EODESCR
1214 };
1215
1216 my $paralleldesc= {
1217         optional => 1,
1218         type => 'string',
1219         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1220         description =>  "Map host parallel devices (n is 0 to 2).",
1221         verbose_description =>  <<EODESCR,
1222 Map host parallel devices (n is 0 to 2).
1223
1224 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1225
1226 CAUTION: Experimental! User reported problems with this option.
1227 EODESCR
1228 };
1229
1230 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1231     $confdesc->{"parallel$i"} = $paralleldesc;
1232 }
1233
1234 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1235     $confdesc->{"serial$i"} = $serialdesc;
1236 }
1237
1238 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1239     $confdesc->{"hostpci$i"} = $hostpcidesc;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1243     $drivename_hash->{"ide$i"} = 1;
1244     $confdesc->{"ide$i"} = $idedesc;
1245 }
1246
1247 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1248     $drivename_hash->{"sata$i"} = 1;
1249     $confdesc->{"sata$i"} = $satadesc;
1250 }
1251
1252 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1253     $drivename_hash->{"scsi$i"} = 1;
1254     $confdesc->{"scsi$i"} = $scsidesc ;
1255 }
1256
1257 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1258     $drivename_hash->{"virtio$i"} = 1;
1259     $confdesc->{"virtio$i"} = $virtiodesc;
1260 }
1261
1262 $drivename_hash->{efidisk0} = 1;
1263 $confdesc->{efidisk0} = $efidisk_desc;
1264
1265 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1266     $confdesc->{"usb$i"} = $usbdesc;
1267 }
1268
1269 my $unuseddesc = {
1270     optional => 1,
1271     type => 'string', format => 'pve-volume-id',
1272     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1273 };
1274
1275 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1276     $confdesc->{"unused$i"} = $unuseddesc;
1277 }
1278
1279 my $kvm_api_version = 0;
1280
1281 sub kvm_version {
1282
1283     return $kvm_api_version if $kvm_api_version;
1284
1285     my $fh = IO::File->new("</dev/kvm") ||
1286         return 0;
1287
1288     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1289         $kvm_api_version = $v;
1290     }
1291
1292     $fh->close();
1293
1294     return  $kvm_api_version;
1295 }
1296
1297 my $kvm_user_version;
1298
1299 sub kvm_user_version {
1300
1301     return $kvm_user_version if $kvm_user_version;
1302
1303     $kvm_user_version = 'unknown';
1304
1305     my $code = sub {
1306         my $line = shift;
1307         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1308             $kvm_user_version = $2;
1309         }
1310     };
1311
1312     eval { run_command("kvm -version", outfunc => $code); };
1313     warn $@ if $@;
1314
1315     return $kvm_user_version;
1316
1317 }
1318
1319 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1320
1321 sub valid_drive_names {
1322     # order is important - used to autoselect boot disk
1323     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1324             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1325             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1326             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1327             'efidisk0');
1328 }
1329
1330 sub is_valid_drivename {
1331     my $dev = shift;
1332
1333     return defined($drivename_hash->{$dev});
1334 }
1335
1336 sub option_exists {
1337     my $key = shift;
1338     return defined($confdesc->{$key});
1339 }
1340
1341 sub nic_models {
1342     return $nic_model_list;
1343 }
1344
1345 sub os_list_description {
1346
1347     return {
1348         other => 'Other',
1349         wxp => 'Windows XP',
1350         w2k => 'Windows 2000',
1351         w2k3 =>, 'Windows 2003',
1352         w2k8 => 'Windows 2008',
1353         wvista => 'Windows Vista',
1354         win7 => 'Windows 7',
1355         win8 => 'Windows 8/2012',
1356         win10 => 'Windows 10/2016',
1357         l24 => 'Linux 2.4',
1358         l26 => 'Linux 2.6',
1359     };
1360 }
1361
1362 my $cdrom_path;
1363
1364 sub get_cdrom_path {
1365
1366     return  $cdrom_path if $cdrom_path;
1367
1368     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1369     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1370     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1371 }
1372
1373 sub get_iso_path {
1374     my ($storecfg, $vmid, $cdrom) = @_;
1375
1376     if ($cdrom eq 'cdrom') {
1377         return get_cdrom_path();
1378     } elsif ($cdrom eq 'none') {
1379         return '';
1380     } elsif ($cdrom =~ m|^/|) {
1381         return $cdrom;
1382     } else {
1383         return PVE::Storage::path($storecfg, $cdrom);
1384     }
1385 }
1386
1387 # try to convert old style file names to volume IDs
1388 sub filename_to_volume_id {
1389     my ($vmid, $file, $media) = @_;
1390
1391      if (!($file eq 'none' || $file eq 'cdrom' ||
1392           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1393
1394         return undef if $file =~ m|/|;
1395
1396         if ($media && $media eq 'cdrom') {
1397             $file = "local:iso/$file";
1398         } else {
1399             $file = "local:$vmid/$file";
1400         }
1401     }
1402
1403     return $file;
1404 }
1405
1406 sub verify_media_type {
1407     my ($opt, $vtype, $media) = @_;
1408
1409     return if !$media;
1410
1411     my $etype;
1412     if ($media eq 'disk') {
1413         $etype = 'images';
1414     } elsif ($media eq 'cdrom') {
1415         $etype = 'iso';
1416     } else {
1417         die "internal error";
1418     }
1419
1420     return if ($vtype eq $etype);
1421
1422     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1423 }
1424
1425 sub cleanup_drive_path {
1426     my ($opt, $storecfg, $drive) = @_;
1427
1428     # try to convert filesystem paths to volume IDs
1429
1430     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1431         ($drive->{file} !~ m|^/dev/.+|) &&
1432         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1433         ($drive->{file} !~ m/^\d+$/)) {
1434         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1435         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1436         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1437         verify_media_type($opt, $vtype, $drive->{media});
1438         $drive->{file} = $volid;
1439     }
1440
1441     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1442 }
1443
1444 sub parse_hotplug_features {
1445     my ($data) = @_;
1446
1447     my $res = {};
1448
1449     return $res if $data eq '0';
1450
1451     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1452
1453     foreach my $feature (PVE::Tools::split_list($data)) {
1454         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1455             $res->{$1} = 1;
1456         } else {
1457             die "invalid hotplug feature '$feature'\n";
1458         }
1459     }
1460     return $res;
1461 }
1462
1463 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1464 sub pve_verify_hotplug_features {
1465     my ($value, $noerr) = @_;
1466
1467     return $value if parse_hotplug_features($value);
1468
1469     return undef if $noerr;
1470
1471     die "unable to parse hotplug option\n";
1472 }
1473
1474 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1475 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1476 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1477 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1478 #        [,iothread=on][,serial=serial][,model=model]
1479
1480 sub parse_drive {
1481     my ($key, $data) = @_;
1482
1483     my ($interface, $index);
1484
1485     if ($key =~ m/^([^\d]+)(\d+)$/) {
1486         $interface = $1;
1487         $index = $2;
1488     } else {
1489         return undef;
1490     }
1491
1492     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1493                                      : $confdesc->{$key}->{format};
1494     if (!$desc) {
1495         warn "invalid drive key: $key\n";
1496         return undef;
1497     }
1498     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1499     return undef if !$res;
1500     $res->{interface} = $interface;
1501     $res->{index} = $index;
1502
1503     my $error = 0;
1504     foreach my $opt (qw(bps bps_rd bps_wr)) {
1505         if (my $bps = defined(delete $res->{$opt})) {
1506             if (defined($res->{"m$opt"})) {
1507                 warn "both $opt and m$opt specified\n";
1508                 ++$error;
1509                 next;
1510             }
1511             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1512         }
1513     }
1514
1515     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1516     for my $requirement (
1517         [mbps_max => 'mbps'],
1518         [mbps_rd_max => 'mbps_rd'],
1519         [mbps_wr_max => 'mbps_wr'],
1520         [miops_max => 'miops'],
1521         [miops_rd_max => 'miops_rd'],
1522         [miops_wr_max => 'miops_wr'],
1523         [bps_max_length => 'mbps_max'],
1524         [bps_rd_max_length => 'mbps_rd_max'],
1525         [bps_wr_max_length => 'mbps_wr_max'],
1526         [iops_max_length => 'iops_max'],
1527         [iops_rd_max_length => 'iops_rd_max'],
1528         [iops_wr_max_length => 'iops_wr_max']) {
1529         my ($option, $requires) = @$requirement;
1530         if ($res->{$option} && !$res->{$requires}) {
1531             warn "$option requires $requires\n";
1532             ++$error;
1533         }
1534     }
1535
1536     return undef if $error;
1537
1538     return undef if $res->{mbps_rd} && $res->{mbps};
1539     return undef if $res->{mbps_wr} && $res->{mbps};
1540     return undef if $res->{iops_rd} && $res->{iops};
1541     return undef if $res->{iops_wr} && $res->{iops};
1542
1543     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1544         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1545         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1546         return undef if $res->{interface} eq 'virtio';
1547     }
1548
1549     if (my $size = $res->{size}) {
1550         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1551     }
1552
1553     return $res;
1554 }
1555
1556 sub print_drive {
1557     my ($vmid, $drive) = @_;
1558     my $data = { %$drive };
1559     delete $data->{$_} for qw(index interface);
1560     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1561 }
1562
1563 sub scsi_inquiry {
1564     my($fh, $noerr) = @_;
1565
1566     my $SG_IO = 0x2285;
1567     my $SG_GET_VERSION_NUM = 0x2282;
1568
1569     my $versionbuf = "\x00" x 8;
1570     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1571     if (!$ret) {
1572         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1573         return undef;
1574     }
1575     my $version = unpack("I", $versionbuf);
1576     if ($version < 30000) {
1577         die "scsi generic interface too old\n"  if !$noerr;
1578         return undef;
1579     }
1580
1581     my $buf = "\x00" x 36;
1582     my $sensebuf = "\x00" x 8;
1583     my $cmd = pack("C x3 C x1", 0x12, 36);
1584
1585     # see /usr/include/scsi/sg.h
1586     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1587
1588     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1589                       length($sensebuf), 0, length($buf), $buf,
1590                       $cmd, $sensebuf, 6000);
1591
1592     $ret = ioctl($fh, $SG_IO, $packet);
1593     if (!$ret) {
1594         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1595         return undef;
1596     }
1597
1598     my @res = unpack($sg_io_hdr_t, $packet);
1599     if ($res[17] || $res[18]) {
1600         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1601         return undef;
1602     }
1603
1604     my $res = {};
1605     (my $byte0, my $byte1, $res->{vendor},
1606      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1607
1608     $res->{removable} = $byte1 & 128 ? 1 : 0;
1609     $res->{type} = $byte0 & 31;
1610
1611     return $res;
1612 }
1613
1614 sub path_is_scsi {
1615     my ($path) = @_;
1616
1617     my $fh = IO::File->new("+<$path") || return undef;
1618     my $res = scsi_inquiry($fh, 1);
1619     close($fh);
1620
1621     return $res;
1622 }
1623
1624 sub machine_type_is_q35 {
1625     my ($conf) = @_;
1626
1627     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1628 }
1629
1630 sub print_tabletdevice_full {
1631     my ($conf) = @_;
1632
1633     my $q35 = machine_type_is_q35($conf);
1634
1635     # we use uhci for old VMs because tablet driver was buggy in older qemu
1636     my $usbbus = $q35 ? "ehci" : "uhci";
1637
1638     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1639 }
1640
1641 sub print_drivedevice_full {
1642     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1643
1644     my $device = '';
1645     my $maxdev = 0;
1646
1647     if ($drive->{interface} eq 'virtio') {
1648         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1649         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1650         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1651     } elsif ($drive->{interface} eq 'scsi') {
1652
1653         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1654         my $unit = $drive->{index} % $maxdev;
1655         my $devicetype = 'hd';
1656         my $path = '';
1657         if (drive_is_cdrom($drive)) {
1658             $devicetype = 'cd';
1659         } else {
1660             if ($drive->{file} =~ m|^/|) {
1661                 $path = $drive->{file};
1662                 if (my $info = path_is_scsi($path)) {
1663                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1664                         $devicetype = 'block';
1665                     } elsif ($info->{type} == 1) { # tape
1666                         $devicetype = 'generic';
1667                     }
1668                 }
1669             } else {
1670                  $path = PVE::Storage::path($storecfg, $drive->{file});
1671             }
1672
1673             if($path =~ m/^iscsi\:\/\//){
1674                 $devicetype = 'generic';
1675             }
1676         }
1677
1678         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1679             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1680         } else {
1681             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682         }
1683
1684     } elsif ($drive->{interface} eq 'ide'){
1685         $maxdev = 2;
1686         my $controller = int($drive->{index} / $maxdev);
1687         my $unit = $drive->{index} % $maxdev;
1688         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1689
1690         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1691         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1692             $model = URI::Escape::uri_unescape($model);
1693             $device .= ",model=$model";
1694         }
1695     } elsif ($drive->{interface} eq 'sata'){
1696         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1697         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1698         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1699     } elsif ($drive->{interface} eq 'usb') {
1700         die "implement me";
1701         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1702     } else {
1703         die "unsupported interface type";
1704     }
1705
1706     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1707
1708     if (my $serial = $drive->{serial}) {
1709         $serial = URI::Escape::uri_unescape($serial);
1710         $device .= ",serial=$serial";
1711     }
1712
1713
1714     return $device;
1715 }
1716
1717 sub get_initiator_name {
1718     my $initiator;
1719
1720     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1721     while (defined(my $line = <$fh>)) {
1722         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1723         $initiator = $1;
1724         last;
1725     }
1726     $fh->close();
1727
1728     return $initiator;
1729 }
1730
1731 sub print_drive_full {
1732     my ($storecfg, $vmid, $drive) = @_;
1733
1734     my $path;
1735     my $volid = $drive->{file};
1736     my $format;
1737
1738     if (drive_is_cdrom($drive)) {
1739         $path = get_iso_path($storecfg, $vmid, $volid);
1740     } else {
1741         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1742         if ($storeid) {
1743             $path = PVE::Storage::path($storecfg, $volid);
1744             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1745             $format = qemu_img_format($scfg, $volname);
1746         } else {
1747             $path = $volid;
1748             $format = "raw";
1749         }
1750    }
1751
1752     my $opts = '';
1753     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1754     foreach my $o (@qemu_drive_options) {
1755         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1756     }
1757
1758     # snapshot only accepts on|off
1759     if (defined($drive->{snapshot})) {
1760         my $v = $drive->{snapshot} ? 'on' : 'off';
1761         $opts .= ",snapshot=$v";
1762     }
1763
1764     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1765         my ($dir, $qmpname) = @$type;
1766         if (my $v = $drive->{"mbps$dir"}) {
1767             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1768         }
1769         if (my $v = $drive->{"mbps${dir}_max"}) {
1770             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1771         }
1772         if (my $v = $drive->{"bps${dir}_max_length"}) {
1773             $opts .= ",throttling.bps$qmpname-max-length=$v";
1774         }
1775         if (my $v = $drive->{"iops${dir}"}) {
1776             $opts .= ",throttling.iops$qmpname=$v";
1777         }
1778         if (my $v = $drive->{"iops${dir}_max"}) {
1779             $opts .= ",throttling.iops$qmpname-max=$v";
1780         }
1781         if (my $v = $drive->{"iops${dir}_max_length"}) {
1782             $opts .= ",throttling.iops$qmpname-max-length=$v";
1783         }
1784     }
1785
1786     $opts .= ",format=$format" if $format && !$drive->{format};
1787
1788     my $cache_direct = 0;
1789
1790     if (my $cache = $drive->{cache}) {
1791         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1792     } elsif (!drive_is_cdrom($drive)) {
1793         $opts .= ",cache=none";
1794         $cache_direct = 1;
1795     }
1796
1797     # aio native works only with O_DIRECT
1798     if (!$drive->{aio}) {
1799         if($cache_direct) {
1800             $opts .= ",aio=native";
1801         } else {
1802             $opts .= ",aio=threads";
1803         }
1804     }
1805
1806     if (!drive_is_cdrom($drive)) {
1807         my $detectzeroes;
1808         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1809             $detectzeroes = 'off';
1810         } elsif ($drive->{discard}) {
1811             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1812         } else {
1813             # This used to be our default with discard not being specified:
1814             $detectzeroes = 'on';
1815         }
1816         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1817     }
1818
1819     my $pathinfo = $path ? "file=$path," : '';
1820
1821     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1822 }
1823
1824 sub print_netdevice_full {
1825     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1826
1827     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1828
1829     my $device = $net->{model};
1830     if ($net->{model} eq 'virtio') {
1831          $device = 'virtio-net-pci';
1832      };
1833
1834     my $pciaddr = print_pci_addr("$netid", $bridges);
1835     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1836     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1837         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1838         my $vectors = $net->{queues} * 2 + 2;
1839         $tmpstr .= ",vectors=$vectors,mq=on";
1840     }
1841     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1842
1843     if ($use_old_bios_files) {
1844         my $romfile;
1845         if ($device eq 'virtio-net-pci') {
1846             $romfile = 'pxe-virtio.rom';
1847         } elsif ($device eq 'e1000') {
1848             $romfile = 'pxe-e1000.rom';
1849         } elsif ($device eq 'ne2k') {
1850             $romfile = 'pxe-ne2k_pci.rom';
1851         } elsif ($device eq 'pcnet') {
1852             $romfile = 'pxe-pcnet.rom';
1853         } elsif ($device eq 'rtl8139') {
1854             $romfile = 'pxe-rtl8139.rom';
1855         }
1856         $tmpstr .= ",romfile=$romfile" if $romfile;
1857     }
1858
1859     return $tmpstr;
1860 }
1861
1862 sub print_netdev_full {
1863     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1864
1865     my $i = '';
1866     if ($netid =~ m/^net(\d+)$/) {
1867         $i = int($1);
1868     }
1869
1870     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1871
1872     my $ifname = "tap${vmid}i$i";
1873
1874     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1875     die "interface name '$ifname' is too long (max 15 character)\n"
1876         if length($ifname) >= 16;
1877
1878     my $vhostparam = '';
1879     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1880
1881     my $vmname = $conf->{name} || "vm$vmid";
1882
1883     my $netdev = "";
1884     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1885
1886     if ($net->{bridge}) {
1887         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1888     } else {
1889         $netdev = "type=user,id=$netid,hostname=$vmname";
1890     }
1891
1892     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1893
1894     return $netdev;
1895 }
1896
1897
1898 sub print_cpu_device {
1899     my ($conf, $id) = @_;
1900
1901     my $kvm = $conf->{kvm} // 1;
1902     my $cpu = $kvm ? "kvm64" : "qemu64";
1903     if (my $cputype = $conf->{cpu}) {
1904         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1905             or die "Cannot parse cpu description: $cputype\n";
1906         $cpu = $cpuconf->{cputype};
1907     }
1908
1909     my $cores = $conf->{cores} || 1;
1910
1911     my $current_core = ($id - 1) % $cores;
1912     my $current_socket = int(($id - 1 - $current_core)/$cores);
1913
1914     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1915 }
1916
1917 sub drive_is_cloudinit {
1918     my ($drive) = @_;
1919     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1920 }
1921
1922 sub drive_is_cdrom {
1923     my ($drive, $exclude_cloudinit) = @_;
1924
1925     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1926
1927     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1928
1929 }
1930
1931 sub parse_number_sets {
1932     my ($set) = @_;
1933     my $res = [];
1934     foreach my $part (split(/;/, $set)) {
1935         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1936             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1937             push @$res, [ $1, $2 ];
1938         } else {
1939             die "invalid range: $part\n";
1940         }
1941     }
1942     return $res;
1943 }
1944
1945 sub parse_numa {
1946     my ($data) = @_;
1947
1948     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1949     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1950     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1951     return $res;
1952 }
1953
1954 sub parse_hostpci {
1955     my ($value) = @_;
1956
1957     return undef if !$value;
1958
1959     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1960
1961     my @idlist = split(/;/, $res->{host});
1962     delete $res->{host};
1963     foreach my $id (@idlist) {
1964         if ($id =~ /^$PCIRE$/) {
1965             if (defined($2)) {
1966                 push @{$res->{pciid}}, { id => $1, function => $2 };
1967             } else {
1968                 my $pcidevices = lspci($1);
1969                 $res->{pciid} = $pcidevices->{$1};
1970             }
1971         } else {
1972             # should have been caught by parse_property_string already
1973             die "failed to parse PCI id: $id\n";
1974         }
1975     }
1976     return $res;
1977 }
1978
1979 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1980 sub parse_net {
1981     my ($data) = @_;
1982
1983     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1984     if ($@) {
1985         warn $@;
1986         return undef;
1987     }
1988     if (!defined($res->{macaddr})) {
1989         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1990         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1991     }
1992     return $res;
1993 }
1994
1995 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1996 sub parse_ipconfig {
1997     my ($data) = @_;
1998
1999     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2000     if ($@) {
2001         warn $@;
2002         return undef;
2003     }
2004
2005     if ($res->{gw} && !$res->{ip}) {
2006         warn 'gateway specified without specifying an IP address';
2007         return undef;
2008     }
2009     if ($res->{gw6} && !$res->{ip6}) {
2010         warn 'IPv6 gateway specified without specifying an IPv6 address';
2011         return undef;
2012     }
2013     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2014         warn 'gateway specified together with DHCP';
2015         return undef;
2016     }
2017     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2018         # gw6 + auto/dhcp
2019         warn "IPv6 gateway specified together with $res->{ip6} address";
2020         return undef;
2021     }
2022
2023     if (!$res->{ip} && !$res->{ip6}) {
2024         return { ip => 'dhcp', ip6 => 'dhcp' };
2025     }
2026
2027     return $res;
2028 }
2029
2030 sub print_net {
2031     my $net = shift;
2032
2033     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2034 }
2035
2036 sub add_random_macs {
2037     my ($settings) = @_;
2038
2039     foreach my $opt (keys %$settings) {
2040         next if $opt !~ m/^net(\d+)$/;
2041         my $net = parse_net($settings->{$opt});
2042         next if !$net;
2043         $settings->{$opt} = print_net($net);
2044     }
2045 }
2046
2047 sub vm_is_volid_owner {
2048     my ($storecfg, $vmid, $volid) = @_;
2049
2050     if ($volid !~  m|^/|) {
2051         my ($path, $owner);
2052         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2053         if ($owner && ($owner == $vmid)) {
2054             return 1;
2055         }
2056     }
2057
2058     return undef;
2059 }
2060
2061 sub split_flagged_list {
2062     my $text = shift || '';
2063     $text =~ s/[,;]/ /g;
2064     $text =~ s/^\s+//;
2065     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2066 }
2067
2068 sub join_flagged_list {
2069     my ($how, $lst) = @_;
2070     join $how, map { $lst->{$_} . $_ } keys %$lst;
2071 }
2072
2073 sub vmconfig_delete_pending_option {
2074     my ($conf, $key, $force) = @_;
2075
2076     delete $conf->{pending}->{$key};
2077     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2078     $pending_delete_hash->{$key} = $force ? '!' : '';
2079     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2080 }
2081
2082 sub vmconfig_undelete_pending_option {
2083     my ($conf, $key) = @_;
2084
2085     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2086     delete $pending_delete_hash->{$key};
2087
2088     if (%$pending_delete_hash) {
2089         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2090     } else {
2091         delete $conf->{pending}->{delete};
2092     }
2093 }
2094
2095 sub vmconfig_register_unused_drive {
2096     my ($storecfg, $vmid, $conf, $drive) = @_;
2097
2098     if (drive_is_cloudinit($drive)) {
2099         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2100         warn $@ if $@;
2101     } elsif (!drive_is_cdrom($drive)) {
2102         my $volid = $drive->{file};
2103         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2104             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2105         }
2106     }
2107 }
2108
2109 sub vmconfig_cleanup_pending {
2110     my ($conf) = @_;
2111
2112     # remove pending changes when nothing changed
2113     my $changes;
2114     foreach my $opt (keys %{$conf->{pending}}) {
2115         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2116             $changes = 1;
2117             delete $conf->{pending}->{$opt};
2118         }
2119     }
2120
2121     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2122     my $pending_delete_hash = {};
2123     while (my ($opt, $force) = each %$current_delete_hash) {
2124         if (defined($conf->{$opt})) {
2125             $pending_delete_hash->{$opt} = $force;
2126         } else {
2127             $changes = 1;
2128         }
2129     }
2130
2131     if (%$pending_delete_hash) {
2132         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2133     } else {
2134         delete $conf->{pending}->{delete};
2135     }
2136
2137     return $changes;
2138 }
2139
2140 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2141 my $smbios1_fmt = {
2142     uuid => {
2143         type => 'string',
2144         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2145         format_description => 'UUID',
2146         description => "Set SMBIOS1 UUID.",
2147         optional => 1,
2148     },
2149     version => {
2150         type => 'string',
2151         pattern => '\S+',
2152         format_description => 'string',
2153         description => "Set SMBIOS1 version.",
2154         optional => 1,
2155     },
2156     serial => {
2157         type => 'string',
2158         pattern => '\S+',
2159         format_description => 'string',
2160         description => "Set SMBIOS1 serial number.",
2161         optional => 1,
2162     },
2163     manufacturer => {
2164         type => 'string',
2165         pattern => '\S+',
2166         format_description => 'string',
2167         description => "Set SMBIOS1 manufacturer.",
2168         optional => 1,
2169     },
2170     product => {
2171         type => 'string',
2172         pattern => '\S+',
2173         format_description => 'string',
2174         description => "Set SMBIOS1 product ID.",
2175         optional => 1,
2176     },
2177     sku => {
2178         type => 'string',
2179         pattern => '\S+',
2180         format_description => 'string',
2181         description => "Set SMBIOS1 SKU string.",
2182         optional => 1,
2183     },
2184     family => {
2185         type => 'string',
2186         pattern => '\S+',
2187         format_description => 'string',
2188         description => "Set SMBIOS1 family string.",
2189         optional => 1,
2190     },
2191 };
2192
2193 sub parse_smbios1 {
2194     my ($data) = @_;
2195
2196     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2197     warn $@ if $@;
2198     return $res;
2199 }
2200
2201 sub print_smbios1 {
2202     my ($smbios1) = @_;
2203     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2204 }
2205
2206 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2207
2208 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2209 sub verify_bootdisk {
2210     my ($value, $noerr) = @_;
2211
2212     return $value if is_valid_drivename($value);
2213
2214     return undef if $noerr;
2215
2216     die "invalid boot disk '$value'\n";
2217 }
2218
2219 sub parse_watchdog {
2220     my ($value) = @_;
2221
2222     return undef if !$value;
2223
2224     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2225     warn $@ if $@;
2226     return $res;
2227 }
2228
2229 sub parse_guest_agent {
2230     my ($value) = @_;
2231
2232     return {} if !defined($value->{agent});
2233
2234     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2235     warn $@ if $@;
2236
2237     # if the agent is disabled ignore the other potentially set properties
2238     return {} if !$res->{enabled};
2239     return $res;
2240 }
2241
2242 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2243 sub verify_usb_device {
2244     my ($value, $noerr) = @_;
2245
2246     return $value if parse_usb_device($value);
2247
2248     return undef if $noerr;
2249
2250     die "unable to parse usb device\n";
2251 }
2252
2253 # add JSON properties for create and set function
2254 sub json_config_properties {
2255     my $prop = shift;
2256
2257     foreach my $opt (keys %$confdesc) {
2258         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2259         $prop->{$opt} = $confdesc->{$opt};
2260     }
2261
2262     return $prop;
2263 }
2264
2265 # return copy of $confdesc_cloudinit to generate documentation
2266 sub cloudinit_config_properties {
2267
2268     return dclone($confdesc_cloudinit);
2269 }
2270
2271 sub check_type {
2272     my ($key, $value) = @_;
2273
2274     die "unknown setting '$key'\n" if !$confdesc->{$key};
2275
2276     my $type = $confdesc->{$key}->{type};
2277
2278     if (!defined($value)) {
2279         die "got undefined value\n";
2280     }
2281
2282     if ($value =~ m/[\n\r]/) {
2283         die "property contains a line feed\n";
2284     }
2285
2286     if ($type eq 'boolean') {
2287         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2288         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2289         die "type check ('boolean') failed - got '$value'\n";
2290     } elsif ($type eq 'integer') {
2291         return int($1) if $value =~ m/^(\d+)$/;
2292         die "type check ('integer') failed - got '$value'\n";
2293     } elsif ($type eq 'number') {
2294         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2295         die "type check ('number') failed - got '$value'\n";
2296     } elsif ($type eq 'string') {
2297         if (my $fmt = $confdesc->{$key}->{format}) {
2298             PVE::JSONSchema::check_format($fmt, $value);
2299             return $value;
2300         }
2301         $value =~ s/^\"(.*)\"$/$1/;
2302         return $value;
2303     } else {
2304         die "internal error"
2305     }
2306 }
2307
2308 sub check_iommu_support{
2309     #fixme : need to check IOMMU support
2310     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2311
2312     my $iommu=1;
2313     return $iommu;
2314
2315 }
2316
2317 sub touch_config {
2318     my ($vmid) = @_;
2319
2320     my $conf = PVE::QemuConfig->config_file($vmid);
2321     utime undef, undef, $conf;
2322 }
2323
2324 sub destroy_vm {
2325     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2326
2327     my $conffile = PVE::QemuConfig->config_file($vmid);
2328
2329     my $conf = PVE::QemuConfig->load_config($vmid);
2330
2331     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2332
2333     if ($conf->{template}) {
2334         # check if any base image is still used by a linked clone
2335         foreach_drive($conf, sub {
2336                 my ($ds, $drive) = @_;
2337
2338                 return if drive_is_cdrom($drive);
2339
2340                 my $volid = $drive->{file};
2341
2342                 return if !$volid || $volid =~ m|^/|;
2343
2344                 die "base volume '$volid' is still in use by linked cloned\n"
2345                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2346
2347         });
2348     }
2349
2350     # only remove disks owned by this VM
2351     foreach_drive($conf, sub {
2352         my ($ds, $drive) = @_;
2353
2354         return if drive_is_cdrom($drive, 1);
2355
2356         my $volid = $drive->{file};
2357
2358         return if !$volid || $volid =~ m|^/|;
2359
2360         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2361         return if !$path || !$owner || ($owner != $vmid);
2362
2363         eval {
2364             PVE::Storage::vdisk_free($storecfg, $volid);
2365         };
2366         warn "Could not remove disk '$volid', check manually: $@" if $@;
2367
2368     });
2369
2370     if ($keep_empty_config) {
2371         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2372     } else {
2373         unlink $conffile;
2374     }
2375
2376     # also remove unused disk
2377     eval {
2378         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2379
2380         eval {
2381             PVE::Storage::foreach_volid($dl, sub {
2382                 my ($volid, $sid, $volname, $d) = @_;
2383                 PVE::Storage::vdisk_free($storecfg, $volid);
2384             });
2385         };
2386         warn $@ if $@;
2387
2388     };
2389     warn $@ if $@;
2390 }
2391
2392 sub parse_vm_config {
2393     my ($filename, $raw) = @_;
2394
2395     return undef if !defined($raw);
2396
2397     my $res = {
2398         digest => Digest::SHA::sha1_hex($raw),
2399         snapshots => {},
2400         pending => {},
2401     };
2402
2403     $filename =~ m|/qemu-server/(\d+)\.conf$|
2404         || die "got strange filename '$filename'";
2405
2406     my $vmid = $1;
2407
2408     my $conf = $res;
2409     my $descr;
2410     my $section = '';
2411
2412     my @lines = split(/\n/, $raw);
2413     foreach my $line (@lines) {
2414         next if $line =~ m/^\s*$/;
2415
2416         if ($line =~ m/^\[PENDING\]\s*$/i) {
2417             $section = 'pending';
2418             if (defined($descr)) {
2419                 $descr =~ s/\s+$//;
2420                 $conf->{description} = $descr;
2421             }
2422             $descr = undef;
2423             $conf = $res->{$section} = {};
2424             next;
2425
2426         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2427             $section = $1;
2428             if (defined($descr)) {
2429                 $descr =~ s/\s+$//;
2430                 $conf->{description} = $descr;
2431             }
2432             $descr = undef;
2433             $conf = $res->{snapshots}->{$section} = {};
2434             next;
2435         }
2436
2437         if ($line =~ m/^\#(.*)\s*$/) {
2438             $descr = '' if !defined($descr);
2439             $descr .= PVE::Tools::decode_text($1) . "\n";
2440             next;
2441         }
2442
2443         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2444             $descr = '' if !defined($descr);
2445             $descr .= PVE::Tools::decode_text($2);
2446         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2447             $conf->{snapstate} = $1;
2448         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2449             my $key = $1;
2450             my $value = $2;
2451             $conf->{$key} = $value;
2452         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2453             my $value = $1;
2454             if ($section eq 'pending') {
2455                 $conf->{delete} = $value; # we parse this later
2456             } else {
2457                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2458             }
2459         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2460             my $key = $1;
2461             my $value = $2;
2462             eval { $value = check_type($key, $value); };
2463             if ($@) {
2464                 warn "vm $vmid - unable to parse value of '$key' - $@";
2465             } else {
2466                 $key = 'ide2' if $key eq 'cdrom';
2467                 my $fmt = $confdesc->{$key}->{format};
2468                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2469                     my $v = parse_drive($key, $value);
2470                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2471                         $v->{file} = $volid;
2472                         $value = print_drive($vmid, $v);
2473                     } else {
2474                         warn "vm $vmid - unable to parse value of '$key'\n";
2475                         next;
2476                     }
2477                 }
2478
2479                 $conf->{$key} = $value;
2480             }
2481         }
2482     }
2483
2484     if (defined($descr)) {
2485         $descr =~ s/\s+$//;
2486         $conf->{description} = $descr;
2487     }
2488     delete $res->{snapstate}; # just to be sure
2489
2490     return $res;
2491 }
2492
2493 sub write_vm_config {
2494     my ($filename, $conf) = @_;
2495
2496     delete $conf->{snapstate}; # just to be sure
2497
2498     if ($conf->{cdrom}) {
2499         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2500         $conf->{ide2} = $conf->{cdrom};
2501         delete $conf->{cdrom};
2502     }
2503
2504     # we do not use 'smp' any longer
2505     if ($conf->{sockets}) {
2506         delete $conf->{smp};
2507     } elsif ($conf->{smp}) {
2508         $conf->{sockets} = $conf->{smp};
2509         delete $conf->{cores};
2510         delete $conf->{smp};
2511     }
2512
2513     my $used_volids = {};
2514
2515     my $cleanup_config = sub {
2516         my ($cref, $pending, $snapname) = @_;
2517
2518         foreach my $key (keys %$cref) {
2519             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2520                 $key eq 'snapstate' || $key eq 'pending';
2521             my $value = $cref->{$key};
2522             if ($key eq 'delete') {
2523                 die "propertry 'delete' is only allowed in [PENDING]\n"
2524                     if !$pending;
2525                 # fixme: check syntax?
2526                 next;
2527             }
2528             eval { $value = check_type($key, $value); };
2529             die "unable to parse value of '$key' - $@" if $@;
2530
2531             $cref->{$key} = $value;
2532
2533             if (!$snapname && is_valid_drivename($key)) {
2534                 my $drive = parse_drive($key, $value);
2535                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2536             }
2537         }
2538     };
2539
2540     &$cleanup_config($conf);
2541
2542     &$cleanup_config($conf->{pending}, 1);
2543
2544     foreach my $snapname (keys %{$conf->{snapshots}}) {
2545         die "internal error" if $snapname eq 'pending';
2546         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2547     }
2548
2549     # remove 'unusedX' settings if we re-add a volume
2550     foreach my $key (keys %$conf) {
2551         my $value = $conf->{$key};
2552         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2553             delete $conf->{$key};
2554         }
2555     }
2556
2557     my $generate_raw_config = sub {
2558         my ($conf, $pending) = @_;
2559
2560         my $raw = '';
2561
2562         # add description as comment to top of file
2563         if (defined(my $descr = $conf->{description})) {
2564             if ($descr) {
2565                 foreach my $cl (split(/\n/, $descr)) {
2566                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2567                 }
2568             } else {
2569                 $raw .= "#\n" if $pending;
2570             }
2571         }
2572
2573         foreach my $key (sort keys %$conf) {
2574             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2575             $raw .= "$key: $conf->{$key}\n";
2576         }
2577         return $raw;
2578     };
2579
2580     my $raw = &$generate_raw_config($conf);
2581
2582     if (scalar(keys %{$conf->{pending}})){
2583         $raw .= "\n[PENDING]\n";
2584         $raw .= &$generate_raw_config($conf->{pending}, 1);
2585     }
2586
2587     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2588         $raw .= "\n[$snapname]\n";
2589         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2590     }
2591
2592     return $raw;
2593 }
2594
2595 sub load_defaults {
2596
2597     my $res = {};
2598
2599     # we use static defaults from our JSON schema configuration
2600     foreach my $key (keys %$confdesc) {
2601         if (defined(my $default = $confdesc->{$key}->{default})) {
2602             $res->{$key} = $default;
2603         }
2604     }
2605
2606     return $res;
2607 }
2608
2609 sub config_list {
2610     my $vmlist = PVE::Cluster::get_vmlist();
2611     my $res = {};
2612     return $res if !$vmlist || !$vmlist->{ids};
2613     my $ids = $vmlist->{ids};
2614
2615     foreach my $vmid (keys %$ids) {
2616         my $d = $ids->{$vmid};
2617         next if !$d->{node} || $d->{node} ne $nodename;
2618         next if !$d->{type} || $d->{type} ne 'qemu';
2619         $res->{$vmid}->{exists} = 1;
2620     }
2621     return $res;
2622 }
2623
2624 # test if VM uses local resources (to prevent migration)
2625 sub check_local_resources {
2626     my ($conf, $noerr) = @_;
2627
2628     my $loc_res = 0;
2629
2630     $loc_res = 1 if $conf->{hostusb}; # old syntax
2631     $loc_res = 1 if $conf->{hostpci}; # old syntax
2632
2633     foreach my $k (keys %$conf) {
2634         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2635         # sockets are safe: they will recreated be on the target side post-migrate
2636         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2637         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2638     }
2639
2640     die "VM uses local resources\n" if $loc_res && !$noerr;
2641
2642     return $loc_res;
2643 }
2644
2645 # check if used storages are available on all nodes (use by migrate)
2646 sub check_storage_availability {
2647     my ($storecfg, $conf, $node) = @_;
2648
2649     foreach_drive($conf, sub {
2650         my ($ds, $drive) = @_;
2651
2652         my $volid = $drive->{file};
2653         return if !$volid;
2654
2655         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2656         return if !$sid;
2657
2658         # check if storage is available on both nodes
2659         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2660         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2661    });
2662 }
2663
2664 # list nodes where all VM images are available (used by has_feature API)
2665 sub shared_nodes {
2666     my ($conf, $storecfg) = @_;
2667
2668     my $nodelist = PVE::Cluster::get_nodelist();
2669     my $nodehash = { map { $_ => 1 } @$nodelist };
2670     my $nodename = PVE::INotify::nodename();
2671
2672     foreach_drive($conf, sub {
2673         my ($ds, $drive) = @_;
2674
2675         my $volid = $drive->{file};
2676         return if !$volid;
2677
2678         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2679         if ($storeid) {
2680             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2681             if ($scfg->{disable}) {
2682                 $nodehash = {};
2683             } elsif (my $avail = $scfg->{nodes}) {
2684                 foreach my $node (keys %$nodehash) {
2685                     delete $nodehash->{$node} if !$avail->{$node};
2686                 }
2687             } elsif (!$scfg->{shared}) {
2688                 foreach my $node (keys %$nodehash) {
2689                     delete $nodehash->{$node} if $node ne $nodename
2690                 }
2691             }
2692         }
2693     });
2694
2695     return $nodehash
2696 }
2697
2698 sub check_cmdline {
2699     my ($pidfile, $pid) = @_;
2700
2701     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2702     if (defined($fh)) {
2703         my $line = <$fh>;
2704         $fh->close;
2705         return undef if !$line;
2706         my @param = split(/\0/, $line);
2707
2708         my $cmd = $param[0];
2709         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2710
2711         for (my $i = 0; $i < scalar (@param); $i++) {
2712             my $p = $param[$i];
2713             next if !$p;
2714             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2715                 my $p = $param[$i+1];
2716                 return 1 if $p && ($p eq $pidfile);
2717                 return undef;
2718             }
2719         }
2720     }
2721     return undef;
2722 }
2723
2724 sub check_running {
2725     my ($vmid, $nocheck, $node) = @_;
2726
2727     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2728
2729     die "unable to find configuration file for VM $vmid - no such machine\n"
2730         if !$nocheck && ! -f $filename;
2731
2732     my $pidfile = pidfile_name($vmid);
2733
2734     if (my $fd = IO::File->new("<$pidfile")) {
2735         my $st = stat($fd);
2736         my $line = <$fd>;
2737         close($fd);
2738
2739         my $mtime = $st->mtime;
2740         if ($mtime > time()) {
2741             warn "file '$filename' modified in future\n";
2742         }
2743
2744         if ($line =~ m/^(\d+)$/) {
2745             my $pid = $1;
2746             if (check_cmdline($pidfile, $pid)) {
2747                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2748                     return $pid;
2749                 }
2750             }
2751         }
2752     }
2753
2754     return undef;
2755 }
2756
2757 sub vzlist {
2758
2759     my $vzlist = config_list();
2760
2761     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2762
2763     while (defined(my $de = $fd->read)) {
2764         next if $de !~ m/^(\d+)\.pid$/;
2765         my $vmid = $1;
2766         next if !defined($vzlist->{$vmid});
2767         if (my $pid = check_running($vmid)) {
2768             $vzlist->{$vmid}->{pid} = $pid;
2769         }
2770     }
2771
2772     return $vzlist;
2773 }
2774
2775 sub disksize {
2776     my ($storecfg, $conf) = @_;
2777
2778     my $bootdisk = $conf->{bootdisk};
2779     return undef if !$bootdisk;
2780     return undef if !is_valid_drivename($bootdisk);
2781
2782     return undef if !$conf->{$bootdisk};
2783
2784     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2785     return undef if !defined($drive);
2786
2787     return undef if drive_is_cdrom($drive);
2788
2789     my $volid = $drive->{file};
2790     return undef if !$volid;
2791
2792     return $drive->{size};
2793 }
2794
2795 our $vmstatus_return_properties = {
2796     vmid => get_standard_option('pve-vmid'),
2797     status => {
2798         description => "Qemu process status.",
2799         type => 'string',
2800         enum => ['stopped', 'running'],
2801     },
2802     maxmem => {
2803         description => "Maximum memory in bytes.",
2804         type => 'integer',
2805         optional => 1,
2806         renderer => 'bytes',
2807     },
2808     maxdisk => {
2809         description => "Root disk size in bytes.",
2810         type => 'integer',
2811         optional => 1,
2812         renderer => 'bytes',
2813     },
2814     name => {
2815         description => "VM name.",
2816         type => 'string',
2817         optional => 1,
2818     },
2819     qmpstatus => {
2820         description => "Qemu QMP agent status.",
2821         type => 'string',
2822         optional => 1,
2823     },
2824     pid => {
2825         description => "PID of running qemu process.",
2826         type => 'integer',
2827         optional => 1,
2828     },
2829     uptime => {
2830         description => "Uptime.",
2831         type => 'integer',
2832         optional => 1,
2833         renderer => 'duration',
2834     },
2835     cpus => {
2836         description => "Maximum usable CPUs.",
2837         type => 'number',
2838         optional => 1,
2839     },
2840 };
2841
2842 my $last_proc_pid_stat;
2843
2844 # get VM status information
2845 # This must be fast and should not block ($full == false)
2846 # We only query KVM using QMP if $full == true (this can be slow)
2847 sub vmstatus {
2848     my ($opt_vmid, $full) = @_;
2849
2850     my $res = {};
2851
2852     my $storecfg = PVE::Storage::config();
2853
2854     my $list = vzlist();
2855     my $defaults = load_defaults();
2856
2857     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2858
2859     my $cpucount = $cpuinfo->{cpus} || 1;
2860
2861     foreach my $vmid (keys %$list) {
2862         next if $opt_vmid && ($vmid ne $opt_vmid);
2863
2864         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2865         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2866
2867         my $d = { vmid => $vmid };
2868         $d->{pid} = $list->{$vmid}->{pid};
2869
2870         # fixme: better status?
2871         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2872
2873         my $size = disksize($storecfg, $conf);
2874         if (defined($size)) {
2875             $d->{disk} = 0; # no info available
2876             $d->{maxdisk} = $size;
2877         } else {
2878             $d->{disk} = 0;
2879             $d->{maxdisk} = 0;
2880         }
2881
2882         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2883             * ($conf->{cores} || $defaults->{cores});
2884         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2885         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2886
2887         $d->{name} = $conf->{name} || "VM $vmid";
2888         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2889             : $defaults->{memory}*(1024*1024);
2890
2891         if ($conf->{balloon}) {
2892             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2893             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2894                 : $defaults->{shares};
2895         }
2896
2897         $d->{uptime} = 0;
2898         $d->{cpu} = 0;
2899         $d->{mem} = 0;
2900
2901         $d->{netout} = 0;
2902         $d->{netin} = 0;
2903
2904         $d->{diskread} = 0;
2905         $d->{diskwrite} = 0;
2906
2907         $d->{template} = PVE::QemuConfig->is_template($conf);
2908
2909         $d->{serial} = 1 if conf_has_serial($conf);
2910
2911         $res->{$vmid} = $d;
2912     }
2913
2914     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2915     foreach my $dev (keys %$netdev) {
2916         next if $dev !~ m/^tap([1-9]\d*)i/;
2917         my $vmid = $1;
2918         my $d = $res->{$vmid};
2919         next if !$d;
2920
2921         $d->{netout} += $netdev->{$dev}->{receive};
2922         $d->{netin} += $netdev->{$dev}->{transmit};
2923
2924         if ($full) {
2925             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2926             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2927         }
2928
2929     }
2930
2931     my $ctime = gettimeofday;
2932
2933     foreach my $vmid (keys %$list) {
2934
2935         my $d = $res->{$vmid};
2936         my $pid = $d->{pid};
2937         next if !$pid;
2938
2939         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2940         next if !$pstat; # not running
2941
2942         my $used = $pstat->{utime} + $pstat->{stime};
2943
2944         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2945
2946         if ($pstat->{vsize}) {
2947             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2948         }
2949
2950         my $old = $last_proc_pid_stat->{$pid};
2951         if (!$old) {
2952             $last_proc_pid_stat->{$pid} = {
2953                 time => $ctime,
2954                 used => $used,
2955                 cpu => 0,
2956             };
2957             next;
2958         }
2959
2960         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2961
2962         if ($dtime > 1000) {
2963             my $dutime = $used -  $old->{used};
2964
2965             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2966             $last_proc_pid_stat->{$pid} = {
2967                 time => $ctime,
2968                 used => $used,
2969                 cpu => $d->{cpu},
2970             };
2971         } else {
2972             $d->{cpu} = $old->{cpu};
2973         }
2974     }
2975
2976     return $res if !$full;
2977
2978     my $qmpclient = PVE::QMPClient->new();
2979
2980     my $ballooncb = sub {
2981         my ($vmid, $resp) = @_;
2982
2983         my $info = $resp->{'return'};
2984         return if !$info->{max_mem};
2985
2986         my $d = $res->{$vmid};
2987
2988         # use memory assigned to VM
2989         $d->{maxmem} = $info->{max_mem};
2990         $d->{balloon} = $info->{actual};
2991
2992         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2993             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2994             $d->{freemem} = $info->{free_mem};
2995         }
2996
2997         $d->{ballooninfo} = $info;
2998     };
2999
3000     my $blockstatscb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} || [];
3003         my $totalrdbytes = 0;
3004         my $totalwrbytes = 0;
3005
3006         for my $blockstat (@$data) {
3007             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3008             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3009
3010             $blockstat->{device} =~ s/drive-//;
3011             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3012         }
3013         $res->{$vmid}->{diskread} = $totalrdbytes;
3014         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3015     };
3016
3017     my $statuscb = sub {
3018         my ($vmid, $resp) = @_;
3019
3020         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3021         # this fails if ballon driver is not loaded, so this must be
3022         # the last commnand (following command are aborted if this fails).
3023         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3024
3025         my $status = 'unknown';
3026         if (!defined($status = $resp->{'return'}->{status})) {
3027             warn "unable to get VM status\n";
3028             return;
3029         }
3030
3031         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3032     };
3033
3034     foreach my $vmid (keys %$list) {
3035         next if $opt_vmid && ($vmid ne $opt_vmid);
3036         next if !$res->{$vmid}->{pid}; # not running
3037         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3038     }
3039
3040     $qmpclient->queue_execute(undef, 2);
3041
3042     foreach my $vmid (keys %$list) {
3043         next if $opt_vmid && ($vmid ne $opt_vmid);
3044         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3045     }
3046
3047     return $res;
3048 }
3049
3050 sub foreach_drive {
3051     my ($conf, $func, @param) = @_;
3052
3053     foreach my $ds (valid_drive_names()) {
3054         next if !defined($conf->{$ds});
3055
3056         my $drive = parse_drive($ds, $conf->{$ds});
3057         next if !$drive;
3058
3059         &$func($ds, $drive, @param);
3060     }
3061 }
3062
3063 sub foreach_volid {
3064     my ($conf, $func, @param) = @_;
3065
3066     my $volhash = {};
3067
3068     my $test_volid = sub {
3069         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3070
3071         return if !$volid;
3072
3073         $volhash->{$volid}->{cdrom} //= 1;
3074         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3075
3076         $volhash->{$volid}->{replicate} //= 0;
3077         $volhash->{$volid}->{replicate} = 1 if $replicate;
3078
3079         $volhash->{$volid}->{shared} //= 0;
3080         $volhash->{$volid}->{shared} = 1 if $shared;
3081
3082         $volhash->{$volid}->{referenced_in_config} //= 0;
3083         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3084
3085         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3086             if defined($snapname);
3087     };
3088
3089     foreach_drive($conf, sub {
3090         my ($ds, $drive) = @_;
3091         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3092     });
3093
3094     foreach my $snapname (keys %{$conf->{snapshots}}) {
3095         my $snap = $conf->{snapshots}->{$snapname};
3096         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3097         foreach_drive($snap, sub {
3098             my ($ds, $drive) = @_;
3099             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3100         });
3101     }
3102
3103     foreach my $volid (keys %$volhash) {
3104         &$func($volid, $volhash->{$volid}, @param);
3105     }
3106 }
3107
3108 sub conf_has_serial {
3109     my ($conf) = @_;
3110
3111     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3112         if ($conf->{"serial$i"}) {
3113             return 1;
3114         }
3115     }
3116
3117     return 0;
3118 }
3119
3120 sub vga_conf_has_spice {
3121     my ($vga) = @_;
3122
3123     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3124
3125     return $1 || 1;
3126 }
3127
3128 sub config_to_command {
3129     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3130
3131     my $cmd = [];
3132     my $globalFlags = [];
3133     my $machineFlags = [];
3134     my $rtcFlags = [];
3135     my $cpuFlags = [];
3136     my $devices = [];
3137     my $pciaddr = '';
3138     my $bridges = {};
3139     my $kvmver = kvm_user_version();
3140     my $vernum = 0; # unknown
3141     my $ostype = $conf->{ostype};
3142     my $winversion = windows_version($ostype);
3143     my $kvm = $conf->{kvm} // 1;
3144
3145     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3146
3147     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3148         $vernum = $1*1000000+$2*1000;
3149     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3150         $vernum = $1*1000000+$2*1000+$3;
3151     }
3152
3153     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3154
3155     my $have_ovz = -f '/proc/vz/vestat';
3156
3157     my $q35 = machine_type_is_q35($conf);
3158     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3159     my $machine_type = $forcemachine || $conf->{machine};
3160     my $use_old_bios_files = undef;
3161     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3162
3163     my $cpuunits = defined($conf->{cpuunits}) ?
3164             $conf->{cpuunits} : $defaults->{cpuunits};
3165
3166     push @$cmd, '/usr/bin/kvm';
3167
3168     push @$cmd, '-id', $vmid;
3169
3170     my $vmname = $conf->{name} || "vm$vmid";
3171
3172     push @$cmd, '-name', $vmname;
3173
3174     my $use_virtio = 0;
3175
3176     my $qmpsocket = qmp_socket($vmid);
3177     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3178     push @$cmd, '-mon', "chardev=qmp,mode=control";
3179
3180
3181     push @$cmd, '-pidfile' , pidfile_name($vmid);
3182
3183     push @$cmd, '-daemonize';
3184
3185     if ($conf->{smbios1}) {
3186         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3187     }
3188
3189     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3190         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3191
3192         my $path;
3193         my $format;
3194         if (my $efidisk = $conf->{efidisk0}) {
3195             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3196             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3197             $format = $d->{format};
3198             if ($storeid) {
3199                 $path = PVE::Storage::path($storecfg, $d->{file});
3200                 if (!defined($format)) {
3201                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3202                     $format = qemu_img_format($scfg, $volname);
3203                 }
3204             } else {
3205                 $path = $d->{file};
3206                 die "efidisk format must be specified\n"
3207                     if !defined($format);
3208             }
3209         } else {
3210             warn "no efidisk configured! Using temporary efivars disk.\n";
3211             $path = "/tmp/$vmid-ovmf.fd";
3212             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3213             $format = 'raw';
3214         }
3215
3216         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3217         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3218     }
3219
3220
3221     # add usb controllers
3222     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3223     push @$devices, @usbcontrollers if @usbcontrollers;
3224     my $vga = $conf->{vga};
3225
3226     my $qxlnum = vga_conf_has_spice($vga);
3227     $vga = 'qxl' if $qxlnum;
3228
3229     if (!$vga) {
3230         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3231             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3232         } else {
3233             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3234         }
3235     }
3236
3237     # enable absolute mouse coordinates (needed by vnc)
3238     my $tablet;
3239     if (defined($conf->{tablet})) {
3240         $tablet = $conf->{tablet};
3241     } else {
3242         $tablet = $defaults->{tablet};
3243         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3244         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3245     }
3246
3247     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3248
3249     my $kvm_off = 0;
3250     my $gpu_passthrough;
3251
3252     # host pci devices
3253     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3254         my $d = parse_hostpci($conf->{"hostpci$i"});
3255         next if !$d;
3256
3257         my $pcie = $d->{pcie};
3258         if($pcie){
3259             die "q35 machine model is not enabled" if !$q35;
3260             $pciaddr = print_pcie_addr("hostpci$i");
3261         }else{
3262             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3263         }
3264
3265         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3266         my $romfile = $d->{romfile};
3267
3268         my $xvga = '';
3269         if ($d->{'x-vga'}) {
3270             $xvga = ',x-vga=on';
3271             $kvm_off = 1;
3272             $vga = 'none';
3273             $gpu_passthrough = 1;
3274
3275             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3276                 $xvga = "";
3277             }
3278         }
3279         my $pcidevices = $d->{pciid};
3280         my $multifunction = 1 if @$pcidevices > 1;
3281
3282         my $j=0;
3283         foreach my $pcidevice (@$pcidevices) {
3284
3285             my $id = "hostpci$i";
3286             $id .= ".$j" if $multifunction;
3287             my $addr = $pciaddr;
3288             $addr .= ".$j" if $multifunction;
3289             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3290
3291             if($j == 0){
3292                 $devicestr .= "$rombar$xvga";
3293                 $devicestr .= ",multifunction=on" if $multifunction;
3294                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3295             }
3296
3297             push @$devices, '-device', $devicestr;
3298             $j++;
3299         }
3300     }
3301
3302     # usb devices
3303     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3304     push @$devices, @usbdevices if @usbdevices;
3305     # serial devices
3306     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3307         if (my $path = $conf->{"serial$i"}) {
3308             if ($path eq 'socket') {
3309                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3310                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3311                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3312             } else {
3313                 die "no such serial device\n" if ! -c $path;
3314                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3315                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3316             }
3317         }
3318     }
3319
3320     # parallel devices
3321     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3322         if (my $path = $conf->{"parallel$i"}) {
3323             die "no such parallel device\n" if ! -c $path;
3324             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3325             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3326             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3327         }
3328     }
3329
3330
3331     my $sockets = 1;
3332     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3333     $sockets = $conf->{sockets} if  $conf->{sockets};
3334
3335     my $cores = $conf->{cores} || 1;
3336
3337     my $maxcpus = $sockets * $cores;
3338
3339     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3340
3341     my $allowed_vcpus = $cpuinfo->{cpus};
3342
3343     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3344         if ($allowed_vcpus < $maxcpus);
3345
3346     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3347
3348         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3349         for (my $i = 2; $i <= $vcpus; $i++)  {
3350             my $cpustr = print_cpu_device($conf,$i);
3351             push @$cmd, '-device', $cpustr;
3352         }
3353
3354     } else {
3355
3356         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3357     }
3358     push @$cmd, '-nodefaults';
3359
3360     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3361
3362     my $bootindex_hash = {};
3363     my $i = 1;
3364     foreach my $o (split(//, $bootorder)) {
3365         $bootindex_hash->{$o} = $i*100;
3366         $i++;
3367     }
3368
3369     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3370
3371     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3372
3373     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3374
3375     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3376
3377     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3378         my $socket = vnc_socket($vmid);
3379         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3380     } else {
3381         push @$cmd, '-nographic';
3382     }
3383
3384     # time drift fix
3385     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3386
3387     my $useLocaltime = $conf->{localtime};
3388
3389     if ($winversion >= 5) { # windows
3390         $useLocaltime = 1 if !defined($conf->{localtime});
3391
3392         # use time drift fix when acpi is enabled
3393         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3394             $tdf = 1 if !defined($conf->{tdf});
3395         }
3396     }
3397
3398     if ($winversion >= 6) {
3399         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3400         push @$cmd, '-no-hpet';
3401     }
3402
3403     push @$rtcFlags, 'driftfix=slew' if $tdf;
3404
3405     if (!$kvm) {
3406         push @$machineFlags, 'accel=tcg';
3407     }
3408
3409     if ($machine_type) {
3410         push @$machineFlags, "type=${machine_type}";
3411     }
3412
3413     if ($conf->{startdate}) {
3414         push @$rtcFlags, "base=$conf->{startdate}";
3415     } elsif ($useLocaltime) {
3416         push @$rtcFlags, 'base=localtime';
3417     }
3418
3419     my $cpu = $kvm ? "kvm64" : "qemu64";
3420     if (my $cputype = $conf->{cpu}) {
3421         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3422             or die "Cannot parse cpu description: $cputype\n";
3423         $cpu = $cpuconf->{cputype};
3424         $kvm_off = 1 if $cpuconf->{hidden};
3425
3426         if (defined(my $flags = $cpuconf->{flags})) {
3427             push @$cpuFlags, split(";", $flags);
3428         }
3429     }
3430
3431     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3432
3433     push @$cpuFlags , '-x2apic'
3434         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3435
3436     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3437
3438     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3439
3440     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3441
3442         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3443         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3444     }
3445
3446     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3447
3448     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3449
3450     push @$cpuFlags, 'kvm=off' if $kvm_off;
3451
3452     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3453         die "internal error"; # should not happen
3454
3455     push @$cpuFlags, "vendor=${cpu_vendor}"
3456         if $cpu_vendor ne 'default';
3457
3458     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3459
3460     push @$cmd, '-cpu', $cpu;
3461
3462     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3463
3464     push @$cmd, '-S' if $conf->{freeze};
3465
3466     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3467
3468     # enable sound
3469     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3470     #push @$cmd, '-soundhw', 'es1370';
3471     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3472
3473     if (parse_guest_agent($conf)->{enabled}) {
3474         my $qgasocket = qmp_socket($vmid, 1);
3475         my $pciaddr = print_pci_addr("qga0", $bridges);
3476         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3477         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3478         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3479     }
3480
3481     my $spice_port;
3482
3483     if ($qxlnum) {
3484         if ($qxlnum > 1) {
3485             if ($winversion){
3486                 for(my $i = 1; $i < $qxlnum; $i++){
3487                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3488                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3489                 }
3490             } else {
3491                 # assume other OS works like Linux
3492                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3493                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3494             }
3495         }
3496
3497         my $pciaddr = print_pci_addr("spice", $bridges);
3498
3499         my $nodename = PVE::INotify::nodename();
3500         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3501         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3502         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3503         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3504         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3505
3506         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3507
3508         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3509         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3510         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3511     }
3512
3513     # enable balloon by default, unless explicitly disabled
3514     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3515         $pciaddr = print_pci_addr("balloon0", $bridges);
3516         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3517     }
3518
3519     if ($conf->{watchdog}) {
3520         my $wdopts = parse_watchdog($conf->{watchdog});
3521         $pciaddr = print_pci_addr("watchdog", $bridges);
3522         my $watchdog = $wdopts->{model} || 'i6300esb';
3523         push @$devices, '-device', "$watchdog$pciaddr";
3524         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3525     }
3526
3527     my $vollist = [];
3528     my $scsicontroller = {};
3529     my $ahcicontroller = {};
3530     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3531
3532     # Add iscsi initiator name if available
3533     if (my $initiator = get_initiator_name()) {
3534         push @$devices, '-iscsi', "initiator-name=$initiator";
3535     }
3536
3537     foreach_drive($conf, sub {
3538         my ($ds, $drive) = @_;
3539
3540         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3541             push @$vollist, $drive->{file};
3542         }
3543
3544         # ignore efidisk here, already added in bios/fw handling code above
3545         return if $drive->{interface} eq 'efidisk';
3546
3547         $use_virtio = 1 if $ds =~ m/^virtio/;
3548
3549         if (drive_is_cdrom ($drive)) {
3550             if ($bootindex_hash->{d}) {
3551                 $drive->{bootindex} = $bootindex_hash->{d};
3552                 $bootindex_hash->{d} += 1;
3553             }
3554         } else {
3555             if ($bootindex_hash->{c}) {
3556                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3557                 $bootindex_hash->{c} += 1;
3558             }
3559         }
3560
3561         if($drive->{interface} eq 'virtio'){
3562            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3563         }
3564
3565         if ($drive->{interface} eq 'scsi') {
3566
3567             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3568
3569             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3570             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3571
3572             my $iothread = '';
3573             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3574                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3575                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3576             } elsif ($drive->{iothread}) {
3577                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3578             }
3579
3580             my $queues = '';
3581             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3582                 $queues = ",num_queues=$drive->{queues}";
3583             }
3584
3585             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3586             $scsicontroller->{$controller}=1;
3587         }
3588
3589         if ($drive->{interface} eq 'sata') {
3590            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3591            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3592            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3593            $ahcicontroller->{$controller}=1;
3594         }
3595
3596         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3597         push @$devices, '-drive',$drive_cmd;
3598         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3599     });
3600
3601     for (my $i = 0; $i < $MAX_NETS; $i++) {
3602          next if !$conf->{"net$i"};
3603          my $d = parse_net($conf->{"net$i"});
3604          next if !$d;
3605
3606          $use_virtio = 1 if $d->{model} eq 'virtio';
3607
3608          if ($bootindex_hash->{n}) {
3609             $d->{bootindex} = $bootindex_hash->{n};
3610             $bootindex_hash->{n} += 1;
3611          }
3612
3613          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3614          push @$devices, '-netdev', $netdevfull;
3615
3616          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3617          push @$devices, '-device', $netdevicefull;
3618     }
3619
3620     if (!$q35) {
3621         # add pci bridges
3622         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3623            $bridges->{1} = 1;
3624            $bridges->{2} = 1;
3625         }
3626
3627         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3628
3629         while (my ($k, $v) = each %$bridges) {
3630             $pciaddr = print_pci_addr("pci.$k");
3631             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3632         }
3633     }
3634
3635     # add custom args
3636     if ($conf->{args}) {
3637         my $aa = PVE::Tools::split_args($conf->{args});
3638         push @$cmd, @$aa;
3639     }
3640
3641     push @$cmd, @$devices;
3642     push @$cmd, '-rtc', join(',', @$rtcFlags)
3643         if scalar(@$rtcFlags);
3644     push @$cmd, '-machine', join(',', @$machineFlags)
3645         if scalar(@$machineFlags);
3646     push @$cmd, '-global', join(',', @$globalFlags)
3647         if scalar(@$globalFlags);
3648
3649     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3650 }
3651
3652 sub vnc_socket {
3653     my ($vmid) = @_;
3654     return "${var_run_tmpdir}/$vmid.vnc";
3655 }
3656
3657 sub spice_port {
3658     my ($vmid) = @_;
3659
3660     my $res = vm_mon_cmd($vmid, 'query-spice');
3661
3662     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3663 }
3664
3665 sub qmp_socket {
3666     my ($vmid, $qga) = @_;
3667     my $sockettype = $qga ? 'qga' : 'qmp';
3668     return "${var_run_tmpdir}/$vmid.$sockettype";
3669 }
3670
3671 sub pidfile_name {
3672     my ($vmid) = @_;
3673     return "${var_run_tmpdir}/$vmid.pid";
3674 }
3675
3676 sub vm_devices_list {
3677     my ($vmid) = @_;
3678
3679     my $res = vm_mon_cmd($vmid, 'query-pci');
3680     my $devices_to_check = [];
3681     my $devices = {};
3682     foreach my $pcibus (@$res) {
3683         push @$devices_to_check, @{$pcibus->{devices}},
3684     }
3685
3686     while (@$devices_to_check) {
3687         my $to_check = [];
3688         for my $d (@$devices_to_check) {
3689             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3690             next if !$d->{'pci_bridge'};
3691
3692             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3693             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3694         }
3695         $devices_to_check = $to_check;
3696     }
3697
3698     my $resblock = vm_mon_cmd($vmid, 'query-block');
3699     foreach my $block (@$resblock) {
3700         if($block->{device} =~ m/^drive-(\S+)/){
3701                 $devices->{$1} = 1;
3702         }
3703     }
3704
3705     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3706     foreach my $mice (@$resmice) {
3707         if ($mice->{name} eq 'QEMU HID Tablet') {
3708             $devices->{tablet} = 1;
3709             last;
3710         }
3711     }
3712
3713     # for usb devices there is no query-usb
3714     # but we can iterate over the entries in
3715     # qom-list path=/machine/peripheral
3716     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3717     foreach my $per (@$resperipheral) {
3718         if ($per->{name} =~ m/^usb\d+$/) {
3719             $devices->{$per->{name}} = 1;
3720         }
3721     }
3722
3723     return $devices;
3724 }
3725
3726 sub vm_deviceplug {
3727     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3728
3729     my $q35 = machine_type_is_q35($conf);
3730
3731     my $devices_list = vm_devices_list($vmid);
3732     return 1 if defined($devices_list->{$deviceid});
3733
3734     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3735
3736     if ($deviceid eq 'tablet') {
3737
3738         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3739
3740     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3741
3742         die "usb hotplug currently not reliable\n";
3743         # since we can't reliably hot unplug all added usb devices
3744         # and usb passthrough disables live migration
3745         # we disable usb hotplugging for now
3746         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3747
3748     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3749
3750         qemu_iothread_add($vmid, $deviceid, $device);
3751
3752         qemu_driveadd($storecfg, $vmid, $device);
3753         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3754
3755         qemu_deviceadd($vmid, $devicefull);
3756         eval { qemu_deviceaddverify($vmid, $deviceid); };
3757         if (my $err = $@) {
3758             eval { qemu_drivedel($vmid, $deviceid); };
3759             warn $@ if $@;
3760             die $err;
3761         }
3762
3763     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3764
3765
3766         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3767         my $pciaddr = print_pci_addr($deviceid);
3768         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3769
3770         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3771
3772         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3773             qemu_iothread_add($vmid, $deviceid, $device);
3774             $devicefull .= ",iothread=iothread-$deviceid";
3775         }
3776
3777         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3778             $devicefull .= ",num_queues=$device->{queues}";
3779         }
3780
3781         qemu_deviceadd($vmid, $devicefull);
3782         qemu_deviceaddverify($vmid, $deviceid);
3783
3784     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3785
3786         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3787         qemu_driveadd($storecfg, $vmid, $device);
3788
3789         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3790         eval { qemu_deviceadd($vmid, $devicefull); };
3791         if (my $err = $@) {
3792             eval { qemu_drivedel($vmid, $deviceid); };
3793             warn $@ if $@;
3794             die $err;
3795         }
3796
3797     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3798
3799         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3800
3801         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3802         my $use_old_bios_files = undef;
3803         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3804
3805         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3806         qemu_deviceadd($vmid, $netdevicefull);
3807         eval { qemu_deviceaddverify($vmid, $deviceid); };
3808         if (my $err = $@) {
3809             eval { qemu_netdevdel($vmid, $deviceid); };
3810             warn $@ if $@;
3811             die $err;
3812         }
3813
3814     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3815
3816         my $bridgeid = $2;
3817         my $pciaddr = print_pci_addr($deviceid);
3818         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3819
3820         qemu_deviceadd($vmid, $devicefull);
3821         qemu_deviceaddverify($vmid, $deviceid);
3822
3823     } else {
3824         die "can't hotplug device '$deviceid'\n";
3825     }
3826
3827     return 1;
3828 }
3829
3830 # fixme: this should raise exceptions on error!
3831 sub vm_deviceunplug {
3832     my ($vmid, $conf, $deviceid) = @_;
3833
3834     my $devices_list = vm_devices_list($vmid);
3835     return 1 if !defined($devices_list->{$deviceid});
3836
3837     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3838
3839     if ($deviceid eq 'tablet') {
3840
3841         qemu_devicedel($vmid, $deviceid);
3842
3843     } elsif ($deviceid =~ m/^usb\d+$/) {
3844
3845         die "usb hotplug currently not reliable\n";
3846         # when unplugging usb devices this way,
3847         # there may be remaining usb controllers/hubs
3848         # so we disable it for now
3849         qemu_devicedel($vmid, $deviceid);
3850         qemu_devicedelverify($vmid, $deviceid);
3851
3852     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3853
3854         qemu_devicedel($vmid, $deviceid);
3855         qemu_devicedelverify($vmid, $deviceid);
3856         qemu_drivedel($vmid, $deviceid);
3857         qemu_iothread_del($conf, $vmid, $deviceid);
3858
3859     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3860
3861         qemu_devicedel($vmid, $deviceid);
3862         qemu_devicedelverify($vmid, $deviceid);
3863         qemu_iothread_del($conf, $vmid, $deviceid);
3864
3865     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3866
3867         qemu_devicedel($vmid, $deviceid);
3868         qemu_drivedel($vmid, $deviceid);
3869         qemu_deletescsihw($conf, $vmid, $deviceid);
3870
3871     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3872
3873         qemu_devicedel($vmid, $deviceid);
3874         qemu_devicedelverify($vmid, $deviceid);
3875         qemu_netdevdel($vmid, $deviceid);
3876
3877     } else {
3878         die "can't unplug device '$deviceid'\n";
3879     }
3880
3881     return 1;
3882 }
3883
3884 sub qemu_deviceadd {
3885     my ($vmid, $devicefull) = @_;
3886
3887     $devicefull = "driver=".$devicefull;
3888     my %options =  split(/[=,]/, $devicefull);
3889
3890     vm_mon_cmd($vmid, "device_add" , %options);
3891 }
3892
3893 sub qemu_devicedel {
3894     my ($vmid, $deviceid) = @_;
3895
3896     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3897 }
3898
3899 sub qemu_iothread_add {
3900     my($vmid, $deviceid, $device) = @_;
3901
3902     if ($device->{iothread}) {
3903         my $iothreads = vm_iothreads_list($vmid);
3904         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3905     }
3906 }
3907
3908 sub qemu_iothread_del {
3909     my($conf, $vmid, $deviceid) = @_;
3910
3911     my $device = parse_drive($deviceid, $conf->{$deviceid});
3912     if ($device->{iothread}) {
3913         my $iothreads = vm_iothreads_list($vmid);
3914         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3915     }
3916 }
3917
3918 sub qemu_objectadd {
3919     my($vmid, $objectid, $qomtype) = @_;
3920
3921     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3922
3923     return 1;
3924 }
3925
3926 sub qemu_objectdel {
3927     my($vmid, $objectid) = @_;
3928
3929     vm_mon_cmd($vmid, "object-del", id => $objectid);
3930
3931     return 1;
3932 }
3933
3934 sub qemu_driveadd {
3935     my ($storecfg, $vmid, $device) = @_;
3936
3937     my $drive = print_drive_full($storecfg, $vmid, $device);
3938     $drive =~ s/\\/\\\\/g;
3939     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3940
3941     # If the command succeeds qemu prints: "OK"
3942     return 1 if $ret =~ m/OK/s;
3943
3944     die "adding drive failed: $ret\n";
3945 }
3946
3947 sub qemu_drivedel {
3948     my($vmid, $deviceid) = @_;
3949
3950     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3951     $ret =~ s/^\s+//;
3952
3953     return 1 if $ret eq "";
3954
3955     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3956     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3957
3958     die "deleting drive $deviceid failed : $ret\n";
3959 }
3960
3961 sub qemu_deviceaddverify {
3962     my ($vmid, $deviceid) = @_;
3963
3964     for (my $i = 0; $i <= 5; $i++) {
3965          my $devices_list = vm_devices_list($vmid);
3966          return 1 if defined($devices_list->{$deviceid});
3967          sleep 1;
3968     }
3969
3970     die "error on hotplug device '$deviceid'\n";
3971 }
3972
3973
3974 sub qemu_devicedelverify {
3975     my ($vmid, $deviceid) = @_;
3976
3977     # need to verify that the device is correctly removed as device_del
3978     # is async and empty return is not reliable
3979
3980     for (my $i = 0; $i <= 5; $i++) {
3981          my $devices_list = vm_devices_list($vmid);
3982          return 1 if !defined($devices_list->{$deviceid});
3983          sleep 1;
3984     }
3985
3986     die "error on hot-unplugging device '$deviceid'\n";
3987 }
3988
3989 sub qemu_findorcreatescsihw {
3990     my ($storecfg, $conf, $vmid, $device) = @_;
3991
3992     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3993
3994     my $scsihwid="$controller_prefix$controller";
3995     my $devices_list = vm_devices_list($vmid);
3996
3997     if(!defined($devices_list->{$scsihwid})) {
3998         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3999     }
4000
4001     return 1;
4002 }
4003
4004 sub qemu_deletescsihw {
4005     my ($conf, $vmid, $opt) = @_;
4006
4007     my $device = parse_drive($opt, $conf->{$opt});
4008
4009     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4010         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4011         return 1;
4012     }
4013
4014     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4015
4016     my $devices_list = vm_devices_list($vmid);
4017     foreach my $opt (keys %{$devices_list}) {
4018         if (PVE::QemuServer::is_valid_drivename($opt)) {
4019             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4020             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4021                 return 1;
4022             }
4023         }
4024     }
4025
4026     my $scsihwid="scsihw$controller";
4027
4028     vm_deviceunplug($vmid, $conf, $scsihwid);
4029
4030     return 1;
4031 }
4032
4033 sub qemu_add_pci_bridge {
4034     my ($storecfg, $conf, $vmid, $device) = @_;
4035
4036     my $bridges = {};
4037
4038     my $bridgeid;
4039
4040     print_pci_addr($device, $bridges);
4041
4042     while (my ($k, $v) = each %$bridges) {
4043         $bridgeid = $k;
4044     }
4045     return 1 if !defined($bridgeid) || $bridgeid < 1;
4046
4047     my $bridge = "pci.$bridgeid";
4048     my $devices_list = vm_devices_list($vmid);
4049
4050     if (!defined($devices_list->{$bridge})) {
4051         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4052     }
4053
4054     return 1;
4055 }
4056
4057 sub qemu_set_link_status {
4058     my ($vmid, $device, $up) = @_;
4059
4060     vm_mon_cmd($vmid, "set_link", name => $device,
4061                up => $up ? JSON::true : JSON::false);
4062 }
4063
4064 sub qemu_netdevadd {
4065     my ($vmid, $conf, $device, $deviceid) = @_;
4066
4067     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4068     my %options =  split(/[=,]/, $netdev);
4069
4070     vm_mon_cmd($vmid, "netdev_add",  %options);
4071     return 1;
4072 }
4073
4074 sub qemu_netdevdel {
4075     my ($vmid, $deviceid) = @_;
4076
4077     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4078 }
4079
4080 sub qemu_usb_hotplug {
4081     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4082
4083     return if !$device;
4084
4085     # remove the old one first
4086     vm_deviceunplug($vmid, $conf, $deviceid);
4087
4088     # check if xhci controller is necessary and available
4089     if ($device->{usb3}) {
4090
4091         my $devicelist = vm_devices_list($vmid);
4092
4093         if (!$devicelist->{xhci}) {
4094             my $pciaddr = print_pci_addr("xhci");
4095             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4096         }
4097     }
4098     my $d = parse_usb_device($device->{host});
4099     $d->{usb3} = $device->{usb3};
4100
4101     # add the new one
4102     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4103 }
4104
4105 sub qemu_cpu_hotplug {
4106     my ($vmid, $conf, $vcpus) = @_;
4107
4108     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4109
4110     my $sockets = 1;
4111     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4112     $sockets = $conf->{sockets} if  $conf->{sockets};
4113     my $cores = $conf->{cores} || 1;
4114     my $maxcpus = $sockets * $cores;
4115
4116     $vcpus = $maxcpus if !$vcpus;
4117
4118     die "you can't add more vcpus than maxcpus\n"
4119         if $vcpus > $maxcpus;
4120
4121     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4122
4123     if ($vcpus < $currentvcpus) {
4124
4125         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4126
4127             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4128                 qemu_devicedel($vmid, "cpu$i");
4129                 my $retry = 0;
4130                 my $currentrunningvcpus = undef;
4131                 while (1) {
4132                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4133                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4134                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4135                     $retry++;
4136                     sleep 1;
4137                 }
4138                 #update conf after each succesfull cpu unplug
4139                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4140                 PVE::QemuConfig->write_config($vmid, $conf);
4141             }
4142         } else {
4143             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4144         }
4145
4146         return;
4147     }
4148
4149     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4150     die "vcpus in running vm does not match its configuration\n"
4151         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4152
4153     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4154
4155         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4156             my $cpustr = print_cpu_device($conf, $i);
4157             qemu_deviceadd($vmid, $cpustr);
4158
4159             my $retry = 0;
4160             my $currentrunningvcpus = undef;
4161             while (1) {
4162                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4163                 last if scalar(@{$currentrunningvcpus}) == $i;
4164                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4165                 sleep 1;
4166                 $retry++;
4167             }
4168             #update conf after each succesfull cpu hotplug
4169             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4170             PVE::QemuConfig->write_config($vmid, $conf);
4171         }
4172     } else {
4173
4174         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4175             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4176         }
4177     }
4178 }
4179
4180 sub qemu_block_set_io_throttle {
4181     my ($vmid, $deviceid,
4182         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4183         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4184         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4185         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4186
4187     return if !check_running($vmid) ;
4188
4189     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4190         bps => int($bps),
4191         bps_rd => int($bps_rd),
4192         bps_wr => int($bps_wr),
4193         iops => int($iops),
4194         iops_rd => int($iops_rd),
4195         iops_wr => int($iops_wr),
4196         bps_max => int($bps_max),
4197         bps_rd_max => int($bps_rd_max),
4198         bps_wr_max => int($bps_wr_max),
4199         iops_max => int($iops_max),
4200         iops_rd_max => int($iops_rd_max),
4201         iops_wr_max => int($iops_wr_max),
4202         bps_max_length => int($bps_max_length),
4203         bps_rd_max_length => int($bps_rd_max_length),
4204         bps_wr_max_length => int($bps_wr_max_length),
4205         iops_max_length => int($iops_max_length),
4206         iops_rd_max_length => int($iops_rd_max_length),
4207         iops_wr_max_length => int($iops_wr_max_length),
4208     );
4209
4210 }
4211
4212 # old code, only used to shutdown old VM after update
4213 sub __read_avail {
4214     my ($fh, $timeout) = @_;
4215
4216     my $sel = new IO::Select;
4217     $sel->add($fh);
4218
4219     my $res = '';
4220     my $buf;
4221
4222     my @ready;
4223     while (scalar (@ready = $sel->can_read($timeout))) {
4224         my $count;
4225         if ($count = $fh->sysread($buf, 8192)) {
4226             if ($buf =~ /^(.*)\(qemu\) $/s) {
4227                 $res .= $1;
4228                 last;
4229             } else {
4230                 $res .= $buf;
4231             }
4232         } else {
4233             if (!defined($count)) {
4234                 die "$!\n";
4235             }
4236             last;
4237         }
4238     }
4239
4240     die "monitor read timeout\n" if !scalar(@ready);
4241
4242     return $res;
4243 }
4244
4245 sub qemu_block_resize {
4246     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4247
4248     my $running = check_running($vmid);
4249
4250     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4251
4252     return if !$running;
4253
4254     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4255
4256 }
4257
4258 sub qemu_volume_snapshot {
4259     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4260
4261     my $running = check_running($vmid);
4262
4263     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4264         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4265     } else {
4266         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4267     }
4268 }
4269
4270 sub qemu_volume_snapshot_delete {
4271     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4272
4273     my $running = check_running($vmid);
4274
4275     if($running) {
4276
4277         $running = undef;
4278         my $conf = PVE::QemuConfig->load_config($vmid);
4279         foreach_drive($conf, sub {
4280             my ($ds, $drive) = @_;
4281             $running = 1 if $drive->{file} eq $volid;
4282         });
4283     }
4284
4285     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4286         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4287     } else {
4288         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4289     }
4290 }
4291
4292 sub set_migration_caps {
4293     my ($vmid) = @_;
4294
4295     my $cap_ref = [];
4296
4297     my $enabled_cap = {
4298         "auto-converge" => 1,
4299         "xbzrle" => 1,
4300         "x-rdma-pin-all" => 0,
4301         "zero-blocks" => 0,
4302         "compress" => 0
4303     };
4304
4305     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4306
4307     for my $supported_capability (@$supported_capabilities) {
4308         push @$cap_ref, {
4309             capability => $supported_capability->{capability},
4310             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4311         };
4312     }
4313
4314     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4315 }
4316
4317 my $fast_plug_option = {
4318     'lock' => 1,
4319     'name' => 1,
4320     'onboot' => 1,
4321     'shares' => 1,
4322     'startup' => 1,
4323     'description' => 1,
4324     'protection' => 1,
4325     'vmstatestorage' => 1,
4326 };
4327
4328 # hotplug changes in [PENDING]
4329 # $selection hash can be used to only apply specified options, for
4330 # example: { cores => 1 } (only apply changed 'cores')
4331 # $errors ref is used to return error messages
4332 sub vmconfig_hotplug_pending {
4333     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4334
4335     my $defaults = load_defaults();
4336
4337     # commit values which do not have any impact on running VM first
4338     # Note: those option cannot raise errors, we we do not care about
4339     # $selection and always apply them.
4340
4341     my $add_error = sub {
4342         my ($opt, $msg) = @_;
4343         $errors->{$opt} = "hotplug problem - $msg";
4344     };
4345
4346     my $changes = 0;
4347     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4348         if ($fast_plug_option->{$opt}) {
4349             $conf->{$opt} = $conf->{pending}->{$opt};
4350             delete $conf->{pending}->{$opt};
4351             $changes = 1;
4352         }
4353     }
4354
4355     if ($changes) {
4356         PVE::QemuConfig->write_config($vmid, $conf);
4357         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4358     }
4359
4360     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4361
4362     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4363     while (my ($opt, $force) = each %$pending_delete_hash) {
4364         next if $selection && !$selection->{$opt};
4365         eval {
4366             if ($opt eq 'hotplug') {
4367                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4368             } elsif ($opt eq 'tablet') {
4369                 die "skip\n" if !$hotplug_features->{usb};
4370                 if ($defaults->{tablet}) {
4371                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4372                 } else {
4373                     vm_deviceunplug($vmid, $conf, $opt);
4374                 }
4375             } elsif ($opt =~ m/^usb\d+/) {
4376                 die "skip\n";
4377                 # since we cannot reliably hot unplug usb devices
4378                 # we are disabling it
4379                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4380                 vm_deviceunplug($vmid, $conf, $opt);
4381             } elsif ($opt eq 'vcpus') {
4382                 die "skip\n" if !$hotplug_features->{cpu};
4383                 qemu_cpu_hotplug($vmid, $conf, undef);
4384             } elsif ($opt eq 'balloon') {
4385                 # enable balloon device is not hotpluggable
4386                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4387                 # here we reset the ballooning value to memory
4388                 my $balloon = $conf->{memory} || $defaults->{memory};
4389                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4390             } elsif ($fast_plug_option->{$opt}) {
4391                 # do nothing
4392             } elsif ($opt =~ m/^net(\d+)$/) {
4393                 die "skip\n" if !$hotplug_features->{network};
4394                 vm_deviceunplug($vmid, $conf, $opt);
4395             } elsif (is_valid_drivename($opt)) {
4396                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4397                 vm_deviceunplug($vmid, $conf, $opt);
4398                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4399             } elsif ($opt =~ m/^memory$/) {
4400                 die "skip\n" if !$hotplug_features->{memory};
4401                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4402             } elsif ($opt eq 'cpuunits') {
4403                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4404             } elsif ($opt eq 'cpulimit') {
4405                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4406             } else {
4407                 die "skip\n";
4408             }
4409         };
4410         if (my $err = $@) {
4411             &$add_error($opt, $err) if $err ne "skip\n";
4412         } else {
4413             # save new config if hotplug was successful
4414             delete $conf->{$opt};
4415             vmconfig_undelete_pending_option($conf, $opt);
4416             PVE::QemuConfig->write_config($vmid, $conf);
4417             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4418         }
4419     }
4420
4421     my $apply_pending_cloudinit;
4422     $apply_pending_cloudinit = sub {
4423         my ($key, $value) = @_;
4424         $apply_pending_cloudinit = sub {}; # once is enough
4425
4426         my @cloudinit_opts = keys %$confdesc_cloudinit;
4427         foreach my $opt (keys %{$conf->{pending}}) {
4428             next if !grep { $_ eq $opt } @cloudinit_opts;
4429             $conf->{$opt} = delete $conf->{pending}->{$opt};
4430         }
4431
4432         my $new_conf = { %$conf };
4433         $new_conf->{$key} = $value;
4434         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4435     };
4436
4437     foreach my $opt (keys %{$conf->{pending}}) {
4438         next if $selection && !$selection->{$opt};
4439         my $value = $conf->{pending}->{$opt};
4440         eval {
4441             if ($opt eq 'hotplug') {
4442                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4443             } elsif ($opt eq 'tablet') {
4444                 die "skip\n" if !$hotplug_features->{usb};
4445                 if ($value == 1) {
4446                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4447                 } elsif ($value == 0) {
4448                     vm_deviceunplug($vmid, $conf, $opt);
4449                 }
4450             } elsif ($opt =~ m/^usb\d+$/) {
4451                 die "skip\n";
4452                 # since we cannot reliably hot unplug usb devices
4453                 # we are disabling it
4454                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4455                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4456                 die "skip\n" if !$d;
4457                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4458             } elsif ($opt eq 'vcpus') {
4459                 die "skip\n" if !$hotplug_features->{cpu};
4460                 qemu_cpu_hotplug($vmid, $conf, $value);
4461             } elsif ($opt eq 'balloon') {
4462                 # enable/disable balloning device is not hotpluggable
4463                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4464                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4465                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4466
4467                 # allow manual ballooning if shares is set to zero
4468                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4469                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4470                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4471                 }
4472             } elsif ($opt =~ m/^net(\d+)$/) {
4473                 # some changes can be done without hotplug
4474                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4475                                     $vmid, $opt, $value);
4476             } elsif (is_valid_drivename($opt)) {
4477                 # some changes can be done without hotplug
4478                 my $drive = parse_drive($opt, $value);
4479                 if (drive_is_cloudinit($drive)) {
4480                     &$apply_pending_cloudinit($opt, $value);
4481                 }
4482                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4483                                      $vmid, $opt, $value, 1);
4484             } elsif ($opt =~ m/^memory$/) { #dimms
4485                 die "skip\n" if !$hotplug_features->{memory};
4486                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4487             } elsif ($opt eq 'cpuunits') {
4488                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4489             } elsif ($opt eq 'cpulimit') {
4490                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4491                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4492             } else {
4493                 die "skip\n";  # skip non-hot-pluggable options
4494             }
4495         };
4496         if (my $err = $@) {
4497             &$add_error($opt, $err) if $err ne "skip\n";
4498         } else {
4499             # save new config if hotplug was successful
4500             $conf->{$opt} = $value;
4501             delete $conf->{pending}->{$opt};
4502             PVE::QemuConfig->write_config($vmid, $conf);
4503             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4504         }
4505     }
4506 }
4507
4508 sub try_deallocate_drive {
4509     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4510
4511     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4512         my $volid = $drive->{file};
4513         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4514             my $sid = PVE::Storage::parse_volume_id($volid);
4515             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4516
4517             # check if the disk is really unused
4518             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4519                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4520             PVE::Storage::vdisk_free($storecfg, $volid);
4521             return 1;
4522         } else {
4523             # If vm is not owner of this disk remove from config
4524             return 1;
4525         }
4526     }
4527
4528     return undef;
4529 }
4530
4531 sub vmconfig_delete_or_detach_drive {
4532     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4533
4534     my $drive = parse_drive($opt, $conf->{$opt});
4535
4536     my $rpcenv = PVE::RPCEnvironment::get();
4537     my $authuser = $rpcenv->get_user();
4538
4539     if ($force) {
4540         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4541         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4542     } else {
4543         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4544     }
4545 }
4546
4547 sub vmconfig_apply_pending {
4548     my ($vmid, $conf, $storecfg) = @_;
4549
4550     # cold plug
4551
4552     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4553     while (my ($opt, $force) = each %$pending_delete_hash) {
4554         die "internal error" if $opt =~ m/^unused/;
4555         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4556         if (!defined($conf->{$opt})) {
4557             vmconfig_undelete_pending_option($conf, $opt);
4558             PVE::QemuConfig->write_config($vmid, $conf);
4559         } elsif (is_valid_drivename($opt)) {
4560             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4561             vmconfig_undelete_pending_option($conf, $opt);
4562             delete $conf->{$opt};
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         } else {
4565             vmconfig_undelete_pending_option($conf, $opt);
4566             delete $conf->{$opt};
4567             PVE::QemuConfig->write_config($vmid, $conf);
4568         }
4569     }
4570
4571     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4572
4573     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4574         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4575
4576         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4577             # skip if nothing changed
4578         } elsif (is_valid_drivename($opt)) {
4579             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4580                 if defined($conf->{$opt});
4581             $conf->{$opt} = $conf->{pending}->{$opt};
4582         } else {
4583             $conf->{$opt} = $conf->{pending}->{$opt};
4584         }
4585
4586         delete $conf->{pending}->{$opt};
4587         PVE::QemuConfig->write_config($vmid, $conf);
4588     }
4589 }
4590
4591 my $safe_num_ne = sub {
4592     my ($a, $b) = @_;
4593
4594     return 0 if !defined($a) && !defined($b);
4595     return 1 if !defined($a);
4596     return 1 if !defined($b);
4597
4598     return $a != $b;
4599 };
4600
4601 my $safe_string_ne = sub {
4602     my ($a, $b) = @_;
4603
4604     return 0 if !defined($a) && !defined($b);
4605     return 1 if !defined($a);
4606     return 1 if !defined($b);
4607
4608     return $a ne $b;
4609 };
4610
4611 sub vmconfig_update_net {
4612     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4613
4614     my $newnet = parse_net($value);
4615
4616     if ($conf->{$opt}) {
4617         my $oldnet = parse_net($conf->{$opt});
4618
4619         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4620             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4621             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4622             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4623
4624             # for non online change, we try to hot-unplug
4625             die "skip\n" if !$hotplug;
4626             vm_deviceunplug($vmid, $conf, $opt);
4627         } else {
4628
4629             die "internal error" if $opt !~ m/net(\d+)/;
4630             my $iface = "tap${vmid}i$1";
4631
4632             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4633                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4634                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4635                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4636                 PVE::Network::tap_unplug($iface);
4637                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4638             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4639                 # Rate can be applied on its own but any change above needs to
4640                 # include the rate in tap_plug since OVS resets everything.
4641                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4642             }
4643
4644             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4645                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4646             }
4647
4648             return 1;
4649         }
4650     }
4651
4652     if ($hotplug) {
4653         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4654     } else {
4655         die "skip\n";
4656     }
4657 }
4658
4659 sub vmconfig_update_disk {
4660     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4661
4662     # fixme: do we need force?
4663
4664     my $drive = parse_drive($opt, $value);
4665
4666     if ($conf->{$opt}) {
4667
4668         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4669
4670             my $media = $drive->{media} || 'disk';
4671             my $oldmedia = $old_drive->{media} || 'disk';
4672             die "unable to change media type\n" if $media ne $oldmedia;
4673
4674             if (!drive_is_cdrom($old_drive)) {
4675
4676                 if ($drive->{file} ne $old_drive->{file}) {
4677
4678                     die "skip\n" if !$hotplug;
4679
4680                     # unplug and register as unused
4681                     vm_deviceunplug($vmid, $conf, $opt);
4682                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4683
4684                 } else {
4685                     # update existing disk
4686
4687                     # skip non hotpluggable value
4688                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4689                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4690                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4691                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4692                         die "skip\n";
4693                     }
4694
4695                     # apply throttle
4696                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4697                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4698                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4699                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4700                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4701                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4702                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4703                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4704                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4705                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4706                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4707                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4708                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4709                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4710                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4711                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4712                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4713                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4714
4715                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4716                                                    ($drive->{mbps} || 0)*1024*1024,
4717                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4718                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4719                                                    $drive->{iops} || 0,
4720                                                    $drive->{iops_rd} || 0,
4721                                                    $drive->{iops_wr} || 0,
4722                                                    ($drive->{mbps_max} || 0)*1024*1024,
4723                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4724                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4725                                                    $drive->{iops_max} || 0,
4726                                                    $drive->{iops_rd_max} || 0,
4727                                                    $drive->{iops_wr_max} || 0,
4728                                                    $drive->{bps_max_length} || 1,
4729                                                    $drive->{bps_rd_max_length} || 1,
4730                                                    $drive->{bps_wr_max_length} || 1,
4731                                                    $drive->{iops_max_length} || 1,
4732                                                    $drive->{iops_rd_max_length} || 1,
4733                                                    $drive->{iops_wr_max_length} || 1);
4734
4735                     }
4736
4737                     return 1;
4738                 }
4739
4740             } else { # cdrom
4741
4742                 if ($drive->{file} eq 'none') {
4743                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4744                     if (drive_is_cloudinit($old_drive)) {
4745                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4746                     }
4747                 } else {
4748                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4749                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4750                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4751                 }
4752
4753                 return 1;
4754             }
4755         }
4756     }
4757
4758     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4759     # hotplug new disks
4760     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4761     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4762 }
4763
4764 sub vm_start {
4765     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4766         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4767
4768     PVE::QemuConfig->lock_config($vmid, sub {
4769         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4770
4771         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4772
4773         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4774
4775         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4776
4777         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4778             vmconfig_apply_pending($vmid, $conf, $storecfg);
4779             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4780         }
4781
4782         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4783
4784         my $defaults = load_defaults();
4785
4786         # set environment variable useful inside network script
4787         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4788
4789         my $local_volumes = {};
4790
4791         if ($targetstorage) {
4792             foreach_drive($conf, sub {
4793                 my ($ds, $drive) = @_;
4794
4795                 return if drive_is_cdrom($drive);
4796
4797                 my $volid = $drive->{file};
4798
4799                 return if !$volid;
4800
4801                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4802
4803                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4804                 return if $scfg->{shared};
4805                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4806             });
4807
4808             my $format = undef;
4809
4810             foreach my $opt (sort keys %$local_volumes) {
4811
4812                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4813                 my $drive = parse_drive($opt, $conf->{$opt});
4814
4815                 #if remote storage is specified, use default format
4816                 if ($targetstorage && $targetstorage ne "1") {
4817                     $storeid = $targetstorage;
4818                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4819                     $format = $defFormat;
4820                 } else {
4821                     #else we use same format than original
4822                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4823                     $format = qemu_img_format($scfg, $volid);
4824                 }
4825
4826                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4827                 my $newdrive = $drive;
4828                 $newdrive->{format} = $format;
4829                 $newdrive->{file} = $newvolid;
4830                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4831                 $local_volumes->{$opt} = $drivestr;
4832                 #pass drive to conf for command line
4833                 $conf->{$opt} = $drivestr;
4834             }
4835         }
4836
4837         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4838
4839         my $migrate_port = 0;
4840         my $migrate_uri;
4841         if ($statefile) {
4842             if ($statefile eq 'tcp') {
4843                 my $localip = "localhost";
4844                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4845                 my $nodename = PVE::INotify::nodename();
4846
4847                 if (!defined($migration_type)) {
4848                     if (defined($datacenterconf->{migration}->{type})) {
4849                         $migration_type = $datacenterconf->{migration}->{type};
4850                     } else {
4851                         $migration_type = 'secure';
4852                     }
4853                 }
4854
4855                 if ($migration_type eq 'insecure') {
4856                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4857                     if ($migrate_network_addr) {
4858                         $localip = $migrate_network_addr;
4859                     } else {
4860                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4861                     }
4862
4863                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4864                 }
4865
4866                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4867                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4868                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4869                 push @$cmd, '-incoming', $migrate_uri;
4870                 push @$cmd, '-S';
4871
4872             } elsif ($statefile eq 'unix') {
4873                 # should be default for secure migrations as a ssh TCP forward
4874                 # tunnel is not deterministic reliable ready and fails regurarly
4875                 # to set up in time, so use UNIX socket forwards
4876                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4877                 unlink $socket_addr;
4878
4879                 $migrate_uri = "unix:$socket_addr";
4880
4881                 push @$cmd, '-incoming', $migrate_uri;
4882                 push @$cmd, '-S';
4883
4884             } else {
4885                 push @$cmd, '-loadstate', $statefile;
4886             }
4887         } elsif ($paused) {
4888             push @$cmd, '-S';
4889         }
4890
4891         # host pci devices
4892         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4893           my $d = parse_hostpci($conf->{"hostpci$i"});
4894           next if !$d;
4895           my $pcidevices = $d->{pciid};
4896           foreach my $pcidevice (@$pcidevices) {
4897                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4898
4899                 my $info = pci_device_info("0000:$pciid");
4900                 die "IOMMU not present\n" if !check_iommu_support();
4901                 die "no pci device info for device '$pciid'\n" if !$info;
4902                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4903                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4904           }
4905         }
4906
4907         PVE::Storage::activate_volumes($storecfg, $vollist);
4908
4909         if (!check_running($vmid, 1)) {
4910             eval {
4911                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4912                     outfunc => sub {}, errfunc => sub {});
4913             };
4914         }
4915
4916         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4917                                                   : $defaults->{cpuunits};
4918
4919         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4920         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4921
4922         my %properties = (
4923             Slice => 'qemu.slice',
4924             KillMode => 'none',
4925             CPUShares => $cpuunits
4926         );
4927
4928         if (my $cpulimit = $conf->{cpulimit}) {
4929             $properties{CPUQuota} = int($cpulimit * 100);
4930         }
4931         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4932
4933         my $run_qemu = sub {
4934             PVE::Tools::run_fork sub {
4935                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4936                 run_command($cmd, %run_params);
4937             };
4938         };
4939
4940         if ($conf->{hugepages}) {
4941
4942             my $code = sub {
4943                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4944                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4945
4946                 PVE::QemuServer::Memory::hugepages_mount();
4947                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4948
4949                 eval { $run_qemu->() };
4950                 if (my $err = $@) {
4951                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4952                     die $err;
4953                 }
4954
4955                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4956             };
4957             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4958
4959         } else {
4960             eval { $run_qemu->() };
4961         }
4962
4963         if (my $err = $@) {
4964             # deactivate volumes if start fails
4965             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4966             die "start failed: $err";
4967         }
4968
4969         print "migration listens on $migrate_uri\n" if $migrate_uri;
4970
4971         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4972             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4973             warn $@ if $@;
4974         }
4975
4976         #start nbd server for storage migration
4977         if ($targetstorage) {
4978             my $nodename = PVE::INotify::nodename();
4979             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4980             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4981             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4982             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4983
4984             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4985
4986             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4987
4988             foreach my $opt (sort keys %$local_volumes) {
4989                 my $volid = $local_volumes->{$opt};
4990                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4991                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4992                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4993             }
4994         }
4995
4996         if ($migratedfrom) {
4997             eval {
4998                 set_migration_caps($vmid);
4999             };
5000             warn $@ if $@;
5001
5002             if ($spice_port) {
5003                 print "spice listens on port $spice_port\n";
5004                 if ($spice_ticket) {
5005                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5006                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5007                 }
5008             }
5009
5010         } else {
5011             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5012                 if !$statefile && $conf->{balloon};
5013
5014             foreach my $opt (keys %$conf) {
5015                 next if $opt !~  m/^net\d+$/;
5016                 my $nicconf = parse_net($conf->{$opt});
5017                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5018             }
5019         }
5020
5021         vm_mon_cmd_nocheck($vmid, 'qom-set',
5022                     path => "machine/peripheral/balloon0",
5023                     property => "guest-stats-polling-interval",
5024                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5025
5026     });
5027 }
5028
5029 sub vm_mon_cmd {
5030     my ($vmid, $execute, %params) = @_;
5031
5032     my $cmd = { execute => $execute, arguments => \%params };
5033     vm_qmp_command($vmid, $cmd);
5034 }
5035
5036 sub vm_mon_cmd_nocheck {
5037     my ($vmid, $execute, %params) = @_;
5038
5039     my $cmd = { execute => $execute, arguments => \%params };
5040     vm_qmp_command($vmid, $cmd, 1);
5041 }
5042
5043 sub vm_qmp_command {
5044     my ($vmid, $cmd, $nocheck) = @_;
5045
5046     my $res;
5047
5048     my $timeout;
5049     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5050         $timeout = $cmd->{arguments}->{timeout};
5051         delete $cmd->{arguments}->{timeout};
5052     }
5053
5054     eval {
5055         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5056         my $sname = qmp_socket($vmid);
5057         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5058             my $qmpclient = PVE::QMPClient->new();
5059
5060             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5061         } else {
5062             die "unable to open monitor socket\n";
5063         }
5064     };
5065     if (my $err = $@) {
5066         syslog("err", "VM $vmid qmp command failed - $err");
5067         die $err;
5068     }
5069
5070     return $res;
5071 }
5072
5073 sub vm_human_monitor_command {
5074     my ($vmid, $cmdline) = @_;
5075
5076     my $res;
5077
5078     my $cmd = {
5079         execute => 'human-monitor-command',
5080         arguments => { 'command-line' => $cmdline},
5081     };
5082
5083     return vm_qmp_command($vmid, $cmd);
5084 }
5085
5086 sub vm_commandline {
5087     my ($storecfg, $vmid) = @_;
5088
5089     my $conf = PVE::QemuConfig->load_config($vmid);
5090
5091     my $defaults = load_defaults();
5092
5093     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5094
5095     return PVE::Tools::cmd2string($cmd);
5096 }
5097
5098 sub vm_reset {
5099     my ($vmid, $skiplock) = @_;
5100
5101     PVE::QemuConfig->lock_config($vmid, sub {
5102
5103         my $conf = PVE::QemuConfig->load_config($vmid);
5104
5105         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5106
5107         vm_mon_cmd($vmid, "system_reset");
5108     });
5109 }
5110
5111 sub get_vm_volumes {
5112     my ($conf) = @_;
5113
5114     my $vollist = [];
5115     foreach_volid($conf, sub {
5116         my ($volid, $attr) = @_;
5117
5118         return if $volid =~ m|^/|;
5119
5120         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5121         return if !$sid;
5122
5123         push @$vollist, $volid;
5124     });
5125
5126     return $vollist;
5127 }
5128
5129 sub vm_stop_cleanup {
5130     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5131
5132     eval {
5133
5134         if (!$keepActive) {
5135             my $vollist = get_vm_volumes($conf);
5136             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5137         }
5138
5139         foreach my $ext (qw(mon qmp pid vnc qga)) {
5140             unlink "/var/run/qemu-server/${vmid}.$ext";
5141         }
5142
5143         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5144     };
5145     warn $@ if $@; # avoid errors - just warn
5146 }
5147
5148 # Note: use $nockeck to skip tests if VM configuration file exists.
5149 # We need that when migration VMs to other nodes (files already moved)
5150 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5151 sub vm_stop {
5152     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5153
5154     $force = 1 if !defined($force) && !$shutdown;
5155
5156     if ($migratedfrom){
5157         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5158         kill 15, $pid if $pid;
5159         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5160         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5161         return;
5162     }
5163
5164     PVE::QemuConfig->lock_config($vmid, sub {
5165
5166         my $pid = check_running($vmid, $nocheck);
5167         return if !$pid;
5168
5169         my $conf;
5170         if (!$nocheck) {
5171             $conf = PVE::QemuConfig->load_config($vmid);
5172             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5173             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5174                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5175                 $timeout = $opts->{down} if $opts->{down};
5176             }
5177         }
5178
5179         $timeout = 60 if !defined($timeout);
5180
5181         eval {
5182             if ($shutdown) {
5183                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5184                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5185                 } else {
5186                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5187                 }
5188             } else {
5189                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5190             }
5191         };
5192         my $err = $@;
5193
5194         if (!$err) {
5195             my $count = 0;
5196             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5197                 $count++;
5198                 sleep 1;
5199             }
5200
5201             if ($count >= $timeout) {
5202                 if ($force) {
5203                     warn "VM still running - terminating now with SIGTERM\n";
5204                     kill 15, $pid;
5205                 } else {
5206                     die "VM quit/powerdown failed - got timeout\n";
5207                 }
5208             } else {
5209                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5210                 return;
5211             }
5212         } else {
5213             if ($force) {
5214                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5215                 kill 15, $pid;
5216             } else {
5217                 die "VM quit/powerdown failed\n";
5218             }
5219         }
5220
5221         # wait again
5222         $timeout = 10;
5223
5224         my $count = 0;
5225         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5226             $count++;
5227             sleep 1;
5228         }
5229
5230         if ($count >= $timeout) {
5231             warn "VM still running - terminating now with SIGKILL\n";
5232             kill 9, $pid;
5233             sleep 1;
5234         }
5235
5236         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5237    });
5238 }
5239
5240 sub vm_suspend {
5241     my ($vmid, $skiplock) = @_;
5242
5243     PVE::QemuConfig->lock_config($vmid, sub {
5244
5245         my $conf = PVE::QemuConfig->load_config($vmid);
5246
5247         PVE::QemuConfig->check_lock($conf)
5248             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5249
5250         vm_mon_cmd($vmid, "stop");
5251     });
5252 }
5253
5254 sub vm_resume {
5255     my ($vmid, $skiplock, $nocheck) = @_;
5256
5257     PVE::QemuConfig->lock_config($vmid, sub {
5258
5259         my $res = vm_mon_cmd($vmid, 'query-status');
5260         my $resume_cmd = 'cont';
5261
5262         if ($res->{status} && $res->{status} eq 'suspended') {
5263             $resume_cmd = 'system_wakeup';
5264         }
5265
5266         if (!$nocheck) {
5267
5268             my $conf = PVE::QemuConfig->load_config($vmid);
5269
5270             PVE::QemuConfig->check_lock($conf)
5271                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5272
5273             vm_mon_cmd($vmid, $resume_cmd);
5274
5275         } else {
5276             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5277         }
5278     });
5279 }
5280
5281 sub vm_sendkey {
5282     my ($vmid, $skiplock, $key) = @_;
5283
5284     PVE::QemuConfig->lock_config($vmid, sub {
5285
5286         my $conf = PVE::QemuConfig->load_config($vmid);
5287
5288         # there is no qmp command, so we use the human monitor command
5289         vm_human_monitor_command($vmid, "sendkey $key");
5290     });
5291 }
5292
5293 sub vm_destroy {
5294     my ($storecfg, $vmid, $skiplock) = @_;
5295
5296     PVE::QemuConfig->lock_config($vmid, sub {
5297
5298         my $conf = PVE::QemuConfig->load_config($vmid);
5299
5300         if (!check_running($vmid)) {
5301             destroy_vm($storecfg, $vmid, undef, $skiplock);
5302         } else {
5303             die "VM $vmid is running - destroy failed\n";
5304         }
5305     });
5306 }
5307
5308 # pci helpers
5309
5310 sub file_write {
5311     my ($filename, $buf) = @_;
5312
5313     my $fh = IO::File->new($filename, "w");
5314     return undef if !$fh;
5315
5316     my $res = print $fh $buf;
5317
5318     $fh->close();
5319
5320     return $res;
5321 }
5322
5323 sub pci_device_info {
5324     my ($name) = @_;
5325
5326     my $res;
5327
5328     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5329     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5330
5331     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5332     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5333
5334     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5335     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5336
5337     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5338     return undef if !defined($product) || $product !~ s/^0x//;
5339
5340     $res = {
5341         name => $name,
5342         vendor => $vendor,
5343         product => $product,
5344         domain => $domain,
5345         bus => $bus,
5346         slot => $slot,
5347         func => $func,
5348         irq => $irq,
5349         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5350     };
5351
5352     return $res;
5353 }
5354
5355 sub pci_dev_reset {
5356     my ($dev) = @_;
5357
5358     my $name = $dev->{name};
5359
5360     my $fn = "$pcisysfs/devices/$name/reset";
5361
5362     return file_write($fn, "1");
5363 }
5364
5365 sub pci_dev_bind_to_vfio {
5366     my ($dev) = @_;
5367
5368     my $name = $dev->{name};
5369
5370     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5371
5372     if (!-d $vfio_basedir) {
5373         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5374     }
5375     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5376
5377     my $testdir = "$vfio_basedir/$name";
5378     return 1 if -d $testdir;
5379
5380     my $data = "$dev->{vendor} $dev->{product}";
5381     return undef if !file_write("$vfio_basedir/new_id", $data);
5382
5383     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5384     if (!file_write($fn, $name)) {
5385         return undef if -f $fn;
5386     }
5387
5388     $fn = "$vfio_basedir/bind";
5389     if (! -d $testdir) {
5390         return undef if !file_write($fn, $name);
5391     }
5392
5393     return -d $testdir;
5394 }
5395
5396 sub pci_dev_group_bind_to_vfio {
5397     my ($pciid) = @_;
5398
5399     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5400
5401     if (!-d $vfio_basedir) {
5402         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5403     }
5404     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5405
5406     # get IOMMU group devices
5407     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5408       my @devs = grep /^0000:/, readdir($D);
5409     closedir($D);
5410
5411     foreach my $pciid (@devs) {
5412         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5413
5414         # pci bridges, switches or root ports are not supported
5415         # they have a pci_bus subdirectory so skip them
5416         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5417
5418         my $info = pci_device_info($1);
5419         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5420     }
5421
5422     return 1;
5423 }
5424
5425 # vzdump restore implementaion
5426
5427 sub tar_archive_read_firstfile {
5428     my $archive = shift;
5429
5430     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5431
5432     # try to detect archive type first
5433     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5434         die "unable to open file '$archive'\n";
5435     my $firstfile = <$fh>;
5436     kill 15, $pid;
5437     close $fh;
5438
5439     die "ERROR: archive contaions no data\n" if !$firstfile;
5440     chomp $firstfile;
5441
5442     return $firstfile;
5443 }
5444
5445 sub tar_restore_cleanup {
5446     my ($storecfg, $statfile) = @_;
5447
5448     print STDERR "starting cleanup\n";
5449
5450     if (my $fd = IO::File->new($statfile, "r")) {
5451         while (defined(my $line = <$fd>)) {
5452             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5453                 my $volid = $2;
5454                 eval {
5455                     if ($volid =~ m|^/|) {
5456                         unlink $volid || die 'unlink failed\n';
5457                     } else {
5458                         PVE::Storage::vdisk_free($storecfg, $volid);
5459                     }
5460                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5461                 };
5462                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5463             } else {
5464                 print STDERR "unable to parse line in statfile - $line";
5465             }
5466         }
5467         $fd->close();
5468     }
5469 }
5470
5471 sub restore_archive {
5472     my ($archive, $vmid, $user, $opts) = @_;
5473
5474     my $format = $opts->{format};
5475     my $comp;
5476
5477     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5478         $format = 'tar' if !$format;
5479         $comp = 'gzip';
5480     } elsif ($archive =~ m/\.tar$/) {
5481         $format = 'tar' if !$format;
5482     } elsif ($archive =~ m/.tar.lzo$/) {
5483         $format = 'tar' if !$format;
5484         $comp = 'lzop';
5485     } elsif ($archive =~ m/\.vma$/) {
5486         $format = 'vma' if !$format;
5487     } elsif ($archive =~ m/\.vma\.gz$/) {
5488         $format = 'vma' if !$format;
5489         $comp = 'gzip';
5490     } elsif ($archive =~ m/\.vma\.lzo$/) {
5491         $format = 'vma' if !$format;
5492         $comp = 'lzop';
5493     } else {
5494         $format = 'vma' if !$format; # default
5495     }
5496
5497     # try to detect archive format
5498     if ($format eq 'tar') {
5499         return restore_tar_archive($archive, $vmid, $user, $opts);
5500     } else {
5501         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5502     }
5503 }
5504
5505 sub restore_update_config_line {
5506     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5507
5508     return if $line =~ m/^\#qmdump\#/;
5509     return if $line =~ m/^\#vzdump\#/;
5510     return if $line =~ m/^lock:/;
5511     return if $line =~ m/^unused\d+:/;
5512     return if $line =~ m/^parent:/;
5513     return if $line =~ m/^template:/; # restored VM is never a template
5514
5515     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5516     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5517         # try to convert old 1.X settings
5518         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5519         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5520             my ($model, $macaddr) = split(/\=/, $devconfig);
5521             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5522             my $net = {
5523                 model => $model,
5524                 bridge => "vmbr$ind",
5525                 macaddr => $macaddr,
5526             };
5527             my $netstr = print_net($net);
5528
5529             print $outfd "net$cookie->{netcount}: $netstr\n";
5530             $cookie->{netcount}++;
5531         }
5532     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5533         my ($id, $netstr) = ($1, $2);
5534         my $net = parse_net($netstr);
5535         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5536         $netstr = print_net($net);
5537         print $outfd "$id: $netstr\n";
5538     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5539         my $virtdev = $1;
5540         my $value = $3;
5541         my $di = parse_drive($virtdev, $value);
5542         if (defined($di->{backup}) && !$di->{backup}) {
5543             print $outfd "#$line";
5544         } elsif ($map->{$virtdev}) {
5545             delete $di->{format}; # format can change on restore
5546             $di->{file} = $map->{$virtdev};
5547             $value = print_drive($vmid, $di);
5548             print $outfd "$virtdev: $value\n";
5549         } else {
5550             print $outfd $line;
5551         }
5552     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5553         my ($uuid, $uuid_str);
5554         UUID::generate($uuid);
5555         UUID::unparse($uuid, $uuid_str);
5556         my $smbios1 = parse_smbios1($2);
5557         $smbios1->{uuid} = $uuid_str;
5558         print $outfd $1.print_smbios1($smbios1)."\n";
5559     } else {
5560         print $outfd $line;
5561     }
5562 }
5563
5564 sub scan_volids {
5565     my ($cfg, $vmid) = @_;
5566
5567     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5568
5569     my $volid_hash = {};
5570     foreach my $storeid (keys %$info) {
5571         foreach my $item (@{$info->{$storeid}}) {
5572             next if !($item->{volid} && $item->{size});
5573             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5574             $volid_hash->{$item->{volid}} = $item;
5575         }
5576     }
5577
5578     return $volid_hash;
5579 }
5580
5581 sub is_volume_in_use {
5582     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5583
5584     my $path = PVE::Storage::path($storecfg, $volid);
5585
5586     my $scan_config = sub {
5587         my ($cref, $snapname) = @_;
5588
5589         foreach my $key (keys %$cref) {
5590             my $value = $cref->{$key};
5591             if (is_valid_drivename($key)) {
5592                 next if $skip_drive && $key eq $skip_drive;
5593                 my $drive = parse_drive($key, $value);
5594                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5595                 return 1 if $volid eq $drive->{file};
5596                 if ($drive->{file} =~ m!^/!) {
5597                     return 1 if $drive->{file} eq $path;
5598                 } else {
5599                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5600                     next if !$storeid;
5601                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5602                     next if !$scfg;
5603                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5604                 }
5605             }
5606         }
5607
5608         return 0;
5609     };
5610
5611     return 1 if &$scan_config($conf);
5612
5613     undef $skip_drive;
5614
5615     foreach my $snapname (keys %{$conf->{snapshots}}) {
5616         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5617     }
5618
5619     return 0;
5620 }
5621
5622 sub update_disksize {
5623     my ($vmid, $conf, $volid_hash) = @_;
5624
5625     my $changes;
5626     my $prefix = "VM $vmid:";
5627
5628     # used and unused disks
5629     my $referenced = {};
5630
5631     # Note: it is allowed to define multiple storages with same path (alias), so
5632     # we need to check both 'volid' and real 'path' (two different volid can point
5633     # to the same path).
5634
5635     my $referencedpath = {};
5636
5637     # update size info
5638     foreach my $opt (keys %$conf) {
5639         if (is_valid_drivename($opt)) {
5640             my $drive = parse_drive($opt, $conf->{$opt});
5641             my $volid = $drive->{file};
5642             next if !$volid;
5643
5644             $referenced->{$volid} = 1;
5645             if ($volid_hash->{$volid} &&
5646                 (my $path = $volid_hash->{$volid}->{path})) {
5647                 $referencedpath->{$path} = 1;
5648             }
5649
5650             next if drive_is_cdrom($drive);
5651             next if !$volid_hash->{$volid};
5652
5653             $drive->{size} = $volid_hash->{$volid}->{size};
5654             my $new = print_drive($vmid, $drive);
5655             if ($new ne $conf->{$opt}) {
5656                 $changes = 1;
5657                 $conf->{$opt} = $new;
5658                 print "$prefix update disk '$opt' information.\n";
5659             }
5660         }
5661     }
5662
5663     # remove 'unusedX' entry if volume is used
5664     foreach my $opt (keys %$conf) {
5665         next if $opt !~ m/^unused\d+$/;
5666         my $volid = $conf->{$opt};
5667         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5668         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5669             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5670             $changes = 1;
5671             delete $conf->{$opt};
5672         }
5673
5674         $referenced->{$volid} = 1;
5675         $referencedpath->{$path} = 1 if $path;
5676     }
5677
5678     foreach my $volid (sort keys %$volid_hash) {
5679         next if $volid =~ m/vm-$vmid-state-/;
5680         next if $referenced->{$volid};
5681         my $path = $volid_hash->{$volid}->{path};
5682         next if !$path; # just to be sure
5683         next if $referencedpath->{$path};
5684         $changes = 1;
5685         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5686         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5687         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5688     }
5689
5690     return $changes;
5691 }
5692
5693 sub rescan {
5694     my ($vmid, $nolock, $dryrun) = @_;
5695
5696     my $cfg = PVE::Storage::config();
5697
5698     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5699     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5700     foreach my $stor (keys %{$cfg->{ids}}) {
5701         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5702     }
5703
5704     print "rescan volumes...\n";
5705     my $volid_hash = scan_volids($cfg, $vmid);
5706
5707     my $updatefn =  sub {
5708         my ($vmid) = @_;
5709
5710         my $conf = PVE::QemuConfig->load_config($vmid);
5711
5712         PVE::QemuConfig->check_lock($conf);
5713
5714         my $vm_volids = {};
5715         foreach my $volid (keys %$volid_hash) {
5716             my $info = $volid_hash->{$volid};
5717             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5718         }
5719
5720         my $changes = update_disksize($vmid, $conf, $vm_volids);
5721
5722         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5723     };
5724
5725     if (defined($vmid)) {
5726         if ($nolock) {
5727             &$updatefn($vmid);
5728         } else {
5729             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5730         }
5731     } else {
5732         my $vmlist = config_list();
5733         foreach my $vmid (keys %$vmlist) {
5734             if ($nolock) {
5735                 &$updatefn($vmid);
5736             } else {
5737                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5738             }
5739         }
5740     }
5741 }
5742
5743 sub restore_vma_archive {
5744     my ($archive, $vmid, $user, $opts, $comp) = @_;
5745
5746     my $readfrom = $archive;
5747
5748     my $cfg = PVE::Storage::config();
5749     my $commands = [];
5750     my $bwlimit = $opts->{bwlimit};
5751
5752     my $dbg_cmdstring = '';
5753     my $add_pipe = sub {
5754         my ($cmd) = @_;
5755         push @$commands, $cmd;
5756         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5757         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5758         $readfrom = '-';
5759     };
5760
5761     my $input = undef;
5762     if ($archive eq '-') {
5763         $input = '<&STDIN';
5764     } else {
5765         # If we use a backup from a PVE defined storage we also consider that
5766         # storage's rate limit:
5767         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5768         if (defined($volid)) {
5769             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5770             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5771             if ($readlimit) {
5772                 print STDERR "applying read rate limit: $readlimit\n";
5773                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5774                 $add_pipe->($cstream);
5775             }
5776         }
5777     }
5778
5779     if ($comp) {
5780         my $cmd;
5781         if ($comp eq 'gzip') {
5782             $cmd = ['zcat', $readfrom];
5783         } elsif ($comp eq 'lzop') {
5784             $cmd = ['lzop', '-d', '-c', $readfrom];
5785         } else {
5786             die "unknown compression method '$comp'\n";
5787         }
5788         $add_pipe->($cmd);
5789     }
5790
5791     my $tmpdir = "/var/tmp/vzdumptmp$$";
5792     rmtree $tmpdir;
5793
5794     # disable interrupts (always do cleanups)
5795     local $SIG{INT} =
5796         local $SIG{TERM} =
5797         local $SIG{QUIT} =
5798         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5799
5800     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5801     POSIX::mkfifo($mapfifo, 0600);
5802     my $fifofh;
5803
5804     my $openfifo = sub {
5805         open($fifofh, '>', $mapfifo) || die $!;
5806     };
5807
5808     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5809
5810     my $oldtimeout;
5811     my $timeout = 5;
5812
5813     my $devinfo = {};
5814
5815     my $rpcenv = PVE::RPCEnvironment::get();
5816
5817     my $conffile = PVE::QemuConfig->config_file($vmid);
5818     my $tmpfn = "$conffile.$$.tmp";
5819
5820     # Note: $oldconf is undef if VM does not exists
5821     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5822     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5823
5824     my %storage_limits;
5825
5826     my $print_devmap = sub {
5827         my $virtdev_hash = {};
5828
5829         my $cfgfn = "$tmpdir/qemu-server.conf";
5830
5831         # we can read the config - that is already extracted
5832         my $fh = IO::File->new($cfgfn, "r") ||
5833             "unable to read qemu-server.conf - $!\n";
5834
5835         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5836         if (-f $fwcfgfn) {
5837             my $pve_firewall_dir = '/etc/pve/firewall';
5838             mkdir $pve_firewall_dir; # make sure the dir exists
5839             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5840         }
5841
5842         while (defined(my $line = <$fh>)) {
5843             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5844                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5845                 die "archive does not contain data for drive '$virtdev'\n"
5846                     if !$devinfo->{$devname};
5847                 if (defined($opts->{storage})) {
5848                     $storeid = $opts->{storage} || 'local';
5849                 } elsif (!$storeid) {
5850                     $storeid = 'local';
5851                 }
5852                 $format = 'raw' if !$format;
5853                 $devinfo->{$devname}->{devname} = $devname;
5854                 $devinfo->{$devname}->{virtdev} = $virtdev;
5855                 $devinfo->{$devname}->{format} = $format;
5856                 $devinfo->{$devname}->{storeid} = $storeid;
5857
5858                 # check permission on storage
5859                 my $pool = $opts->{pool}; # todo: do we need that?
5860                 if ($user ne 'root@pam') {
5861                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5862                 }
5863
5864                 $storage_limits{$storeid} = $bwlimit;
5865
5866                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5867             }
5868         }
5869
5870         foreach my $key (keys %storage_limits) {
5871             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5872             next if !$limit;
5873             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5874             $storage_limits{$key} = $limit * 1024;
5875         }
5876
5877         foreach my $devname (keys %$devinfo) {
5878             die "found no device mapping information for device '$devname'\n"
5879                 if !$devinfo->{$devname}->{virtdev};
5880         }
5881
5882         # create empty/temp config
5883         if ($oldconf) {
5884             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5885             foreach_drive($oldconf, sub {
5886                 my ($ds, $drive) = @_;
5887
5888                 return if drive_is_cdrom($drive);
5889
5890                 my $volid = $drive->{file};
5891
5892                 return if !$volid || $volid =~ m|^/|;
5893
5894                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5895                 return if !$path || !$owner || ($owner != $vmid);
5896
5897                 # Note: only delete disk we want to restore
5898                 # other volumes will become unused
5899                 if ($virtdev_hash->{$ds}) {
5900                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5901                     if (my $err = $@) {
5902                         warn $err;
5903                     }
5904                 }
5905             });
5906
5907             # delete vmstate files
5908             # since after the restore we have no snapshots anymore
5909             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5910                 my $snap = $oldconf->{snapshots}->{$snapname};
5911                 if ($snap->{vmstate}) {
5912                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5913                     if (my $err = $@) {
5914                         warn $err;
5915                     }
5916                 }
5917             }
5918         }
5919
5920         my $map = {};
5921         foreach my $virtdev (sort keys %$virtdev_hash) {
5922             my $d = $virtdev_hash->{$virtdev};
5923             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5924             my $storeid = $d->{storeid};
5925             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5926
5927             my $map_opts = '';
5928             if (my $limit = $storage_limits{$storeid}) {
5929                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5930             }
5931
5932             # test if requested format is supported
5933             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5934             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5935             $d->{format} = $defFormat if !$supported;
5936
5937             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5938                                                   $d->{format}, undef, $alloc_size);
5939             print STDERR "new volume ID is '$volid'\n";
5940             $d->{volid} = $volid;
5941             my $path = PVE::Storage::path($cfg, $volid);
5942
5943             PVE::Storage::activate_volumes($cfg,[$volid]);
5944
5945             my $write_zeros = 1;
5946             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5947                 $write_zeros = 0;
5948             }
5949
5950             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5951
5952             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5953             $map->{$virtdev} = $volid;
5954         }
5955
5956         $fh->seek(0, 0) || die "seek failed - $!\n";
5957
5958         my $outfd = new IO::File ($tmpfn, "w") ||
5959             die "unable to write config for VM $vmid\n";
5960
5961         my $cookie = { netcount => 0 };
5962         while (defined(my $line = <$fh>)) {
5963             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5964         }
5965
5966         $fh->close();
5967         $outfd->close();
5968     };
5969
5970     eval {
5971         # enable interrupts
5972         local $SIG{INT} =
5973             local $SIG{TERM} =
5974             local $SIG{QUIT} =
5975             local $SIG{HUP} =
5976             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5977         local $SIG{ALRM} = sub { die "got timeout\n"; };
5978
5979         $oldtimeout = alarm($timeout);
5980
5981         my $parser = sub {
5982             my $line = shift;
5983
5984             print "$line\n";
5985
5986             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5987                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5988                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5989             } elsif ($line =~ m/^CTIME: /) {
5990                 # we correctly received the vma config, so we can disable
5991                 # the timeout now for disk allocation (set to 10 minutes, so
5992                 # that we always timeout if something goes wrong)
5993                 alarm(600);
5994                 &$print_devmap();
5995                 print $fifofh "done\n";
5996                 my $tmp = $oldtimeout || 0;
5997                 $oldtimeout = undef;
5998                 alarm($tmp);
5999                 close($fifofh);
6000             }
6001         };
6002
6003         print "restore vma archive: $dbg_cmdstring\n";
6004         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6005     };
6006     my $err = $@;
6007
6008     alarm($oldtimeout) if $oldtimeout;
6009
6010     my $vollist = [];
6011     foreach my $devname (keys %$devinfo) {
6012         my $volid = $devinfo->{$devname}->{volid};
6013         push @$vollist, $volid if $volid;
6014     }
6015
6016     PVE::Storage::deactivate_volumes($cfg, $vollist);
6017
6018     unlink $mapfifo;
6019
6020     if ($err) {
6021         rmtree $tmpdir;
6022         unlink $tmpfn;
6023
6024         foreach my $devname (keys %$devinfo) {
6025             my $volid = $devinfo->{$devname}->{volid};
6026             next if !$volid;
6027             eval {
6028                 if ($volid =~ m|^/|) {
6029                     unlink $volid || die 'unlink failed\n';
6030                 } else {
6031                     PVE::Storage::vdisk_free($cfg, $volid);
6032                 }
6033                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6034             };
6035             print STDERR "unable to cleanup '$volid' - $@" if $@;
6036         }
6037         die $err;
6038     }
6039
6040     rmtree $tmpdir;
6041
6042     rename($tmpfn, $conffile) ||
6043         die "unable to commit configuration file '$conffile'\n";
6044
6045     PVE::Cluster::cfs_update(); # make sure we read new file
6046
6047     eval { rescan($vmid, 1); };
6048     warn $@ if $@;
6049 }
6050
6051 sub restore_tar_archive {
6052     my ($archive, $vmid, $user, $opts) = @_;
6053
6054     if ($archive ne '-') {
6055         my $firstfile = tar_archive_read_firstfile($archive);
6056         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6057             if $firstfile ne 'qemu-server.conf';
6058     }
6059
6060     my $storecfg = PVE::Storage::config();
6061
6062     # destroy existing data - keep empty config
6063     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6064     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6065
6066     my $tocmd = "/usr/lib/qemu-server/qmextract";
6067
6068     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6069     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6070     $tocmd .= ' --prealloc' if $opts->{prealloc};
6071     $tocmd .= ' --info' if $opts->{info};
6072
6073     # tar option "xf" does not autodetect compression when read from STDIN,
6074     # so we pipe to zcat
6075     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6076         PVE::Tools::shellquote("--to-command=$tocmd");
6077
6078     my $tmpdir = "/var/tmp/vzdumptmp$$";
6079     mkpath $tmpdir;
6080
6081     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6082     local $ENV{VZDUMP_VMID} = $vmid;
6083     local $ENV{VZDUMP_USER} = $user;
6084
6085     my $conffile = PVE::QemuConfig->config_file($vmid);
6086     my $tmpfn = "$conffile.$$.tmp";
6087
6088     # disable interrupts (always do cleanups)
6089     local $SIG{INT} =
6090         local $SIG{TERM} =
6091         local $SIG{QUIT} =
6092         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6093
6094     eval {
6095         # enable interrupts
6096         local $SIG{INT} =
6097             local $SIG{TERM} =
6098             local $SIG{QUIT} =
6099             local $SIG{HUP} =
6100             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6101
6102         if ($archive eq '-') {
6103             print "extracting archive from STDIN\n";
6104             run_command($cmd, input => "<&STDIN");
6105         } else {
6106             print "extracting archive '$archive'\n";
6107             run_command($cmd);
6108         }
6109
6110         return if $opts->{info};
6111
6112         # read new mapping
6113         my $map = {};
6114         my $statfile = "$tmpdir/qmrestore.stat";
6115         if (my $fd = IO::File->new($statfile, "r")) {
6116             while (defined (my $line = <$fd>)) {
6117                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6118                     $map->{$1} = $2 if $1;
6119                 } else {
6120                     print STDERR "unable to parse line in statfile - $line\n";
6121                 }
6122             }
6123             $fd->close();
6124         }
6125
6126         my $confsrc = "$tmpdir/qemu-server.conf";
6127
6128         my $srcfd = new IO::File($confsrc, "r") ||
6129             die "unable to open file '$confsrc'\n";
6130
6131         my $outfd = new IO::File ($tmpfn, "w") ||
6132             die "unable to write config for VM $vmid\n";
6133
6134         my $cookie = { netcount => 0 };
6135         while (defined (my $line = <$srcfd>)) {
6136             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6137         }
6138
6139         $srcfd->close();
6140         $outfd->close();
6141     };
6142     my $err = $@;
6143
6144     if ($err) {
6145
6146         unlink $tmpfn;
6147
6148         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6149
6150         die $err;
6151     }
6152
6153     rmtree $tmpdir;
6154
6155     rename $tmpfn, $conffile ||
6156         die "unable to commit configuration file '$conffile'\n";
6157
6158     PVE::Cluster::cfs_update(); # make sure we read new file
6159
6160     eval { rescan($vmid, 1); };
6161     warn $@ if $@;
6162 };
6163
6164 sub foreach_storage_used_by_vm {
6165     my ($conf, $func) = @_;
6166
6167     my $sidhash = {};
6168
6169     foreach_drive($conf, sub {
6170         my ($ds, $drive) = @_;
6171         return if drive_is_cdrom($drive);
6172
6173         my $volid = $drive->{file};
6174
6175         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6176         $sidhash->{$sid} = $sid if $sid;
6177     });
6178
6179     foreach my $sid (sort keys %$sidhash) {
6180         &$func($sid);
6181     }
6182 }
6183
6184 sub do_snapshots_with_qemu {
6185     my ($storecfg, $volid) = @_;
6186
6187     my $storage_name = PVE::Storage::parse_volume_id($volid);
6188
6189     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6190         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6191         return 1;
6192     }
6193
6194     if ($volid =~ m/\.(qcow2|qed)$/){
6195         return 1;
6196     }
6197
6198     return undef;
6199 }
6200
6201 sub qga_check_running {
6202     my ($vmid, $nowarn) = @_;
6203
6204     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6205     if ($@) {
6206         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6207         return 0;
6208     }
6209     return 1;
6210 }
6211
6212 sub template_create {
6213     my ($vmid, $conf, $disk) = @_;
6214
6215     my $storecfg = PVE::Storage::config();
6216
6217     foreach_drive($conf, sub {
6218         my ($ds, $drive) = @_;
6219
6220         return if drive_is_cdrom($drive);
6221         return if $disk && $ds ne $disk;
6222
6223         my $volid = $drive->{file};
6224         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6225
6226         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6227         $drive->{file} = $voliddst;
6228         $conf->{$ds} = print_drive($vmid, $drive);
6229         PVE::QemuConfig->write_config($vmid, $conf);
6230     });
6231 }
6232
6233 sub qemu_img_convert {
6234     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6235
6236     my $storecfg = PVE::Storage::config();
6237     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6238     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6239
6240     if ($src_storeid && $dst_storeid) {
6241
6242         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6243
6244         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6245         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6246
6247         my $src_format = qemu_img_format($src_scfg, $src_volname);
6248         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6249
6250         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6251         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6252
6253         my $cmd = [];
6254         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6255         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6256         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6257         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6258         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6259         if ($is_zero_initialized) {
6260             push @$cmd, "zeroinit:$dst_path";
6261         } else {
6262             push @$cmd, $dst_path;
6263         }
6264
6265         my $parser = sub {
6266             my $line = shift;
6267             if($line =~ m/\((\S+)\/100\%\)/){
6268                 my $percent = $1;
6269                 my $transferred = int($size * $percent / 100);
6270                 my $remaining = $size - $transferred;
6271
6272                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6273             }
6274
6275         };
6276
6277         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6278         my $err = $@;
6279         die "copy failed: $err" if $err;
6280     }
6281 }
6282
6283 sub qemu_img_format {
6284     my ($scfg, $volname) = @_;
6285
6286     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6287         return $1;
6288     } else {
6289         return "raw";
6290     }
6291 }
6292
6293 sub qemu_drive_mirror {
6294     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6295
6296     $jobs = {} if !$jobs;
6297
6298     my $qemu_target;
6299     my $format;
6300     $jobs->{"drive-$drive"} = {};
6301
6302     if ($dst_volid =~ /^nbd:/) {
6303         $qemu_target = $dst_volid;
6304         $format = "nbd";
6305     } else {
6306         my $storecfg = PVE::Storage::config();
6307         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6308
6309         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6310
6311         $format = qemu_img_format($dst_scfg, $dst_volname);
6312
6313         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6314
6315         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6316     }
6317
6318     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6319     $opts->{format} = $format if $format;
6320
6321     print "drive mirror is starting for drive-$drive\n";
6322
6323     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6324
6325     if (my $err = $@) {
6326         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6327         die "mirroring error: $err";
6328     }
6329
6330     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6331 }
6332
6333 sub qemu_drive_mirror_monitor {
6334     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6335
6336     eval {
6337         my $err_complete = 0;
6338
6339         while (1) {
6340             die "storage migration timed out\n" if $err_complete > 300;
6341
6342             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6343
6344             my $running_mirror_jobs = {};
6345             foreach my $stat (@$stats) {
6346                 next if $stat->{type} ne 'mirror';
6347                 $running_mirror_jobs->{$stat->{device}} = $stat;
6348             }
6349
6350             my $readycounter = 0;
6351
6352             foreach my $job (keys %$jobs) {
6353
6354                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6355                     print "$job : finished\n";
6356                     delete $jobs->{$job};
6357                     next;
6358                 }
6359
6360                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6361
6362                 my $busy = $running_mirror_jobs->{$job}->{busy};
6363                 my $ready = $running_mirror_jobs->{$job}->{ready};
6364                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6365                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6366                     my $remaining = $total - $transferred;
6367                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6368
6369                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6370                 }
6371
6372                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6373             }
6374
6375             last if scalar(keys %$jobs) == 0;
6376
6377             if ($readycounter == scalar(keys %$jobs)) {
6378                 print "all mirroring jobs are ready \n";
6379                 last if $skipcomplete; #do the complete later
6380
6381                 if ($vmiddst && $vmiddst != $vmid) {
6382                     my $agent_running = $qga && qga_check_running($vmid);
6383                     if ($agent_running) {
6384                         print "freeze filesystem\n";
6385                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6386                     } else {
6387                         print "suspend vm\n";
6388                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6389                     }
6390
6391                     # if we clone a disk for a new target vm, we don't switch the disk
6392                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6393
6394                     if ($agent_running) {
6395                         print "unfreeze filesystem\n";
6396                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6397                     } else {
6398                         print "resume vm\n";
6399                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6400                     }
6401
6402                     last;
6403                 } else {
6404
6405                     foreach my $job (keys %$jobs) {
6406                         # try to switch the disk if source and destination are on the same guest
6407                         print "$job: Completing block job...\n";
6408
6409                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6410                         if ($@ =~ m/cannot be completed/) {
6411                             print "$job: Block job cannot be completed, try again.\n";
6412                             $err_complete++;
6413                         }else {
6414                             print "$job: Completed successfully.\n";
6415                             $jobs->{$job}->{complete} = 1;
6416                         }
6417                     }
6418                 }
6419             }
6420             sleep 1;
6421         }
6422     };
6423     my $err = $@;
6424
6425     if ($err) {
6426         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6427         die "mirroring error: $err";
6428     }
6429
6430 }
6431
6432 sub qemu_blockjobs_cancel {
6433     my ($vmid, $jobs) = @_;
6434
6435     foreach my $job (keys %$jobs) {
6436         print "$job: Cancelling block job\n";
6437         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6438         $jobs->{$job}->{cancel} = 1;
6439     }
6440
6441     while (1) {
6442         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6443
6444         my $running_jobs = {};
6445         foreach my $stat (@$stats) {
6446             $running_jobs->{$stat->{device}} = $stat;
6447         }
6448
6449         foreach my $job (keys %$jobs) {
6450
6451             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6452                 print "$job: Done.\n";
6453                 delete $jobs->{$job};
6454             }
6455         }
6456
6457         last if scalar(keys %$jobs) == 0;
6458
6459         sleep 1;
6460     }
6461 }
6462
6463 sub clone_disk {
6464     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6465         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6466
6467     my $newvolid;
6468
6469     if (!$full) {
6470         print "create linked clone of drive $drivename ($drive->{file})\n";
6471         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6472         push @$newvollist, $newvolid;
6473     } else {
6474
6475         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6476         $storeid = $storage if $storage;
6477
6478         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6479         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6480
6481         print "create full clone of drive $drivename ($drive->{file})\n";
6482         my $name = undef;
6483         if (drive_is_cloudinit($drive)) {
6484             $name = "vm-$newvmid-cloudinit";
6485             # cloudinit only supports raw and qcow2 atm:
6486             if ($dst_format eq 'qcow2') {
6487                 $name .= '.qcow2';
6488             } elsif ($dst_format ne 'raw') {
6489                 die "clone: unhandled format for cloudinit image\n";
6490             }
6491         }
6492         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6493         push @$newvollist, $newvolid;
6494
6495         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6496
6497         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6498         if (!$running || $snapname) {
6499             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6500         } else {
6501
6502             my $kvmver = get_running_qemu_version ($vmid);
6503             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6504                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6505                     if $drive->{iothread};
6506             }
6507
6508             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6509         }
6510     }
6511
6512     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6513
6514     my $disk = $drive;
6515     $disk->{format} = undef;
6516     $disk->{file} = $newvolid;
6517     $disk->{size} = $size;
6518
6519     return $disk;
6520 }
6521
6522 # this only works if VM is running
6523 sub get_current_qemu_machine {
6524     my ($vmid) = @_;
6525
6526     my $cmd = { execute => 'query-machines', arguments => {} };
6527     my $res = vm_qmp_command($vmid, $cmd);
6528
6529     my ($current, $default);
6530     foreach my $e (@$res) {
6531         $default = $e->{name} if $e->{'is-default'};
6532         $current = $e->{name} if $e->{'is-current'};
6533     }
6534
6535     # fallback to the default machine if current is not supported by qemu
6536     return $current || $default || 'pc';
6537 }
6538
6539 sub get_running_qemu_version {
6540     my ($vmid) = @_;
6541     my $cmd = { execute => 'query-version', arguments => {} };
6542     my $res = vm_qmp_command($vmid, $cmd);
6543     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6544 }
6545
6546 sub qemu_machine_feature_enabled {
6547     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6548
6549     my $current_major;
6550     my $current_minor;
6551
6552     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6553
6554         $current_major = $3;
6555         $current_minor = $4;
6556
6557     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6558
6559         $current_major = $1;
6560         $current_minor = $2;
6561     }
6562
6563     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6564
6565
6566 }
6567
6568 sub qemu_machine_pxe {
6569     my ($vmid, $conf, $machine) = @_;
6570
6571     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6572
6573     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6574         $machine .= '.pxe';
6575     }
6576
6577     return $machine;
6578 }
6579
6580 sub qemu_use_old_bios_files {
6581     my ($machine_type) = @_;
6582
6583     return if !$machine_type;
6584
6585     my $use_old_bios_files = undef;
6586
6587     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6588         $machine_type = $1;
6589         $use_old_bios_files = 1;
6590     } else {
6591         my $kvmver = kvm_user_version();
6592         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6593         # load new efi bios files on migration. So this hack is required to allow
6594         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6595         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6596         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6597     }
6598
6599     return ($use_old_bios_files, $machine_type);
6600 }
6601
6602 sub create_efidisk {
6603     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6604
6605     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6606
6607     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6608     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6609     PVE::Storage::activate_volumes($storecfg, [$volid]);
6610
6611     my $path = PVE::Storage::path($storecfg, $volid);
6612     eval {
6613         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6614     };
6615     die "Copying EFI vars image failed: $@" if $@;
6616
6617     return ($volid, $vars_size);
6618 }
6619
6620 sub lspci {
6621
6622     my $devices = {};
6623
6624     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6625             my (undef, $id, $function) = @_;
6626             my $res = { id => $id, function => $function};
6627             push @{$devices->{$id}}, $res;
6628     });
6629
6630     # Entries should be sorted by functions.
6631     foreach my $id (keys %$devices) {
6632         my $dev = $devices->{$id};
6633         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6634     }
6635
6636     return $devices;
6637 }
6638
6639 sub vm_iothreads_list {
6640     my ($vmid) = @_;
6641
6642     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6643
6644     my $iothreads = {};
6645     foreach my $iothread (@$res) {
6646         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6647     }
6648
6649     return $iothreads;
6650 }
6651
6652 sub scsihw_infos {
6653     my ($conf, $drive) = @_;
6654
6655     my $maxdev = 0;
6656
6657     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6658         $maxdev = 7;
6659     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6660         $maxdev = 1;
6661     } else {
6662         $maxdev = 256;
6663     }
6664
6665     my $controller = int($drive->{index} / $maxdev);
6666     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6667
6668     return ($maxdev, $controller, $controller_prefix);
6669 }
6670
6671 sub add_hyperv_enlightenments {
6672     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6673
6674     return if $winversion < 6;
6675     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6676
6677     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6678
6679     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6680         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6681         push @$cpuFlags , 'hv_vapic';
6682         push @$cpuFlags , 'hv_time';
6683     } else {
6684         push @$cpuFlags , 'hv_spinlocks=0xffff';
6685     }
6686
6687     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6688         push @$cpuFlags , 'hv_reset';
6689         push @$cpuFlags , 'hv_vpindex';
6690         push @$cpuFlags , 'hv_runtime';
6691     }
6692
6693     if ($winversion >= 7) {
6694         push @$cpuFlags , 'hv_relaxed';
6695     }
6696 }
6697
6698 sub windows_version {
6699     my ($ostype) = @_;
6700
6701     return 0 if !$ostype;
6702
6703     my $winversion = 0;
6704
6705     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6706         $winversion = 5;
6707     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6708         $winversion = 6;
6709     } elsif ($ostype =~ m/^win(\d+)$/) {
6710         $winversion = $1;
6711     }
6712
6713     return $winversion;
6714 }
6715
6716 sub resolve_dst_disk_format {
6717         my ($storecfg, $storeid, $src_volname, $format) = @_;
6718         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6719
6720         if (!$format) {
6721             # if no target format is specified, use the source disk format as hint
6722             if ($src_volname) {
6723                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6724                 $format = qemu_img_format($scfg, $src_volname);
6725             } else {
6726                 return $defFormat;
6727             }
6728         }
6729
6730         # test if requested format is supported - else use default
6731         my $supported = grep { $_ eq $format } @$validFormats;
6732         $format = $defFormat if !$supported;
6733         return $format;
6734 }
6735
6736 sub resolve_first_disk {
6737     my $conf = shift;
6738     my @disks = PVE::QemuServer::valid_drive_names();
6739     my $firstdisk;
6740     foreach my $ds (reverse @disks) {
6741         next if !$conf->{$ds};
6742         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6743         next if PVE::QemuServer::drive_is_cdrom($disk);
6744         $firstdisk = $ds;
6745     }
6746     return $firstdisk;
6747 }
6748
6749 sub generate_smbios1_uuid {
6750     my ($uuid, $uuid_str);
6751     UUID::generate($uuid);
6752     UUID::unparse($uuid, $uuid_str);
6753     return "uuid=$uuid_str";
6754 }
6755
6756 # bash completion helper
6757
6758 sub complete_backup_archives {
6759     my ($cmdname, $pname, $cvalue) = @_;
6760
6761     my $cfg = PVE::Storage::config();
6762
6763     my $storeid;
6764
6765     if ($cvalue =~ m/^([^:]+):/) {
6766         $storeid = $1;
6767     }
6768
6769     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6770
6771     my $res = [];
6772     foreach my $id (keys %$data) {
6773         foreach my $item (@{$data->{$id}}) {
6774             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6775             push @$res, $item->{volid} if defined($item->{volid});
6776         }
6777     }
6778
6779     return $res;
6780 }
6781
6782 my $complete_vmid_full = sub {
6783     my ($running) = @_;
6784
6785     my $idlist = vmstatus();
6786
6787     my $res = [];
6788
6789     foreach my $id (keys %$idlist) {
6790         my $d = $idlist->{$id};
6791         if (defined($running)) {
6792             next if $d->{template};
6793             next if $running && $d->{status} ne 'running';
6794             next if !$running && $d->{status} eq 'running';
6795         }
6796         push @$res, $id;
6797
6798     }
6799     return $res;
6800 };
6801
6802 sub complete_vmid {
6803     return &$complete_vmid_full();
6804 }
6805
6806 sub complete_vmid_stopped {
6807     return &$complete_vmid_full(0);
6808 }
6809
6810 sub complete_vmid_running {
6811     return &$complete_vmid_full(1);
6812 }
6813
6814 sub complete_storage {
6815
6816     my $cfg = PVE::Storage::config();
6817     my $ids = $cfg->{ids};
6818
6819     my $res = [];
6820     foreach my $sid (keys %$ids) {
6821         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6822         next if !$ids->{$sid}->{content}->{images};
6823         push @$res, $sid;
6824     }
6825
6826     return $res;
6827 }
6828
6829 sub nbd_stop {
6830     my ($vmid) = @_;
6831
6832     vm_mon_cmd($vmid, 'nbd-server-stop');
6833 }
6834
6835 1;