PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 #no warnings 'redefine';
  84
  85 sub cgroups_write {
  86    my ($controller, $vmid, $option, $value) = @_;
  87
  88    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  89    PVE::ProcFSTools::write_proc_entry($path, $value);
  90
  91 }
  92
  93 my $nodename = PVE::INotify::nodename();
  94
  95 mkdir "/etc/pve/nodes/$nodename";
  96 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  97 mkdir $confdir;
  98
  99 my $var_run_tmpdir = "/var/run/qemu-server";
 100 mkdir $var_run_tmpdir;
 101
 102 my $lock_dir = "/var/lock/qemu-server";
 103 mkdir $lock_dir;
 104
 105 my $pcisysfs = "/sys/bus/pci";
 106
 107 my $cpu_vendor_list = {
 108     # Intel CPUs
 109     486 => 'GenuineIntel',
 110     pentium => 'GenuineIntel',
 111     pentium2  => 'GenuineIntel',
 112     pentium3  => 'GenuineIntel',
 113     coreduo => 'GenuineIntel',
 114     core2duo => 'GenuineIntel',
 115     Conroe  => 'GenuineIntel',
 116     Penryn  => 'GenuineIntel',
 117     Nehalem  => 'GenuineIntel',
 118     'Nehalem-IBRS'  => 'GenuineIntel',
 119     Westmere => 'GenuineIntel',
 120     'Westmere-IBRS' => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     'SandyBridge-IBRS' => 'GenuineIntel',
 123     IvyBridge => 'GenuineIntel',
 124     'IvyBridge-IBRS' => 'GenuineIntel',
 125     Haswell => 'GenuineIntel',
 126     'Haswell-IBRS' => 'GenuineIntel',
 127     'Haswell-noTSX' => 'GenuineIntel',
 128     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 129     Broadwell => 'GenuineIntel',
 130     'Broadwell-IBRS' => 'GenuineIntel',
 131     'Broadwell-noTSX' => 'GenuineIntel',
 132     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 133     'Skylake-Client' => 'GenuineIntel',
 134     'Skylake-Client-IBRS' => 'GenuineIntel',
 135     'Skylake-Server' => 'GenuineIntel',
 136     'Skylake-Server-IBRS' => 'GenuineIntel',
 137
 138     # AMD CPUs
 139     athlon => 'AuthenticAMD',
 140     phenom  => 'AuthenticAMD',
 141     Opteron_G1  => 'AuthenticAMD',
 142     Opteron_G2  => 'AuthenticAMD',
 143     Opteron_G3  => 'AuthenticAMD',
 144     Opteron_G4  => 'AuthenticAMD',
 145     Opteron_G5  => 'AuthenticAMD',
 146     EPYC => 'AuthenticAMD',
 147     'EPYC-IBPB' => 'AuthenticAMD',
 148
 149     # generic types, use vendor from host node
 150     host => 'default',
 151     kvm32 => 'default',
 152     kvm64 => 'default',
 153     qemu32 => 'default',
 154     qemu64 => 'default',
 155     max => 'default',
 156 };
 157
 158 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 159
 160 my $cpu_fmt = {
 161     cputype => {
 162         description => "Emulated CPU type.",
 163         type => 'string',
 164         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 165         default => 'kvm64',
 166         default_key => 1,
 167     },
 168     hidden => {
 169         description => "Do not identify as a KVM virtual machine.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 0
 173     },
 174     flags => {
 175         description => "List of additional CPU flags separated by ';'."
 176                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 177                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 178         format_description => '+FLAG[;-FLAG...]',
 179         type => 'string',
 180         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 181         optional => 1,
 182     },
 183 };
 184
 185 my $watchdog_fmt = {
 186     model => {
 187         default_key => 1,
 188         type => 'string',
 189         enum => [qw(i6300esb ib700)],
 190         description => "Watchdog type to emulate.",
 191         default => 'i6300esb',
 192         optional => 1,
 193     },
 194     action => {
 195         type => 'string',
 196         enum => [qw(reset shutdown poweroff pause debug none)],
 197         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 198         optional => 1,
 199     },
 200 };
 201 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 202
 203 my $confdesc = {
 204     onboot => {
 205         optional => 1,
 206         type => 'boolean',
 207         description => "Specifies whether a VM will be started during system bootup.",
 208         default => 0,
 209     },
 210     autostart => {
 211         optional => 1,
 212         type => 'boolean',
 213         description => "Automatic restart after crash (currently ignored).",
 214         default => 0,
 215     },
 216     hotplug => {
 217         optional => 1,
 218         type => 'string', format => 'pve-hotplug-features',
 219         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 220         default => 'network,disk,usb',
 221     },
 222     reboot => {
 223         optional => 1,
 224         type => 'boolean',
 225         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 226         default => 1,
 227     },
 228     lock => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Lock/unlock the VM.",
 232         enum => [qw(migrate backup snapshot rollback)],
 233     },
 234     cpulimit => {
 235         optional => 1,
 236         type => 'number',
 237         description => "Limit of CPU usage.",
 238         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 239         minimum => 0,
 240         maximum => 128,
 241         default => 0,
 242     },
 243     cpuunits => {
 244         optional => 1,
 245         type => 'integer',
 246         description => "CPU weight for a VM.",
 247         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 248         minimum => 2,
 249         maximum => 262144,
 250         default => 1024,
 251     },
 252     memory => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 256         minimum => 16,
 257         default => 512,
 258     },
 259     balloon => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 263         minimum => 0,
 264     },
 265     shares => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 269         minimum => 0,
 270         maximum => 50000,
 271         default => 1000,
 272     },
 273     keyboard => {
 274         optional => 1,
 275         type => 'string',
 276         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 277                        "It should not be necessary to set it.",
 278         enum => PVE::Tools::kvmkeymaplist(),
 279         default => undef,
 280     },
 281     name => {
 282         optional => 1,
 283         type => 'string', format => 'dns-name',
 284         description => "Set a name for the VM. Only used on the configuration web interface.",
 285     },
 286     scsihw => {
 287         optional => 1,
 288         type => 'string',
 289         description => "SCSI controller model",
 290         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 291         default => 'lsi',
 292     },
 293     description => {
 294         optional => 1,
 295         type => 'string',
 296         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 297     },
 298     ostype => {
 299         optional => 1,
 300         type => 'string',
 301         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 302         description => "Specify guest operating system.",
 303         verbose_description => <<EODESC,
 304 Specify guest operating system. This is used to enable special
 305 optimization/features for specific operating systems:
 306
 307 [horizontal]
 308 other;; unspecified OS
 309 wxp;; Microsoft Windows XP
 310 w2k;; Microsoft Windows 2000
 311 w2k3;; Microsoft Windows 2003
 312 w2k8;; Microsoft Windows 2008
 313 wvista;; Microsoft Windows Vista
 314 win7;; Microsoft Windows 7
 315 win8;; Microsoft Windows 8/2012/2012r2
 316 win10;; Microsoft Windows 10/2016
 317 l24;; Linux 2.4 Kernel
 318 l26;; Linux 2.6/3.X Kernel
 319 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 320 EODESC
 321     },
 322     boot => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 326         pattern => '[acdn]{1,4}',
 327         default => 'cdn',
 328     },
 329     bootdisk => {
 330         optional => 1,
 331         type => 'string', format => 'pve-qm-bootdisk',
 332         description => "Enable booting from specified disk.",
 333         pattern => '(ide|sata|scsi|virtio)\d+',
 334     },
 335     smp => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPUs. Please use option -sockets instead.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     sockets => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of CPU sockets.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     cores => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "The number of cores per socket.",
 353         minimum => 1,
 354         default => 1,
 355     },
 356     numa => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable NUMA.",
 360         default => 0,
 361     },
 362     hugepages => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Enable/disable hugepages memory.",
 366         enum => [qw(any 2 1024)],
 367     },
 368     vcpus => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Number of hotplugged vcpus.",
 372         minimum => 1,
 373         default => 0,
 374     },
 375     acpi => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable ACPI.",
 379         default => 1,
 380     },
 381     agent => {
 382         optional => 1,
 383         type => 'boolean',
 384         description => "Enable/disable Qemu GuestAgent.",
 385         default => 0,
 386     },
 387     kvm => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Enable/disable KVM hardware virtualization.",
 391         default => 1,
 392     },
 393     tdf => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Enable/disable time drift fix.",
 397         default => 0,
 398     },
 399     localtime => {
 400         optional => 1,
 401         type => 'boolean',
 402         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 403     },
 404     freeze => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 408     },
 409     vga => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Select the VGA type.",
 413         verbose_description => "Select the VGA type. If you want to use high resolution" .
 414             " modes (>= 1280x1024x16) then you should use the options " .
 415             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 416             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 417             "display sever. For win* OS you can select how many independent " .
 418             "displays you want, Linux guests can add displays them self. " .
 419             "You can also run without any graphic card, using a serial device" .
 420             " as terminal.",
 421         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 422     },
 423     watchdog => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-watchdog',
 426         description => "Create a virtual hardware watchdog device.",
 427         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 428             " (by a guest action), the watchdog must be periodically polled " .
 429             "by an agent inside the guest or else the watchdog will reset " .
 430             "the guest (or execute the respective action specified)",
 431     },
 432     startdate => {
 433         optional => 1,
 434         type => 'string',
 435         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 436         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 437         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 438         default => 'now',
 439     },
 440     startup =>  get_standard_option('pve-startup-order'),
 441     template => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable Template.",
 445         default => 0,
 446     },
 447     args => {
 448         optional => 1,
 449         type => 'string',
 450         description => "Arbitrary arguments passed to kvm.",
 451         verbose_description => <<EODESCR,
 452 Arbitrary arguments passed to kvm, for example:
 453
 454 args: -no-reboot -no-hpet
 455
 456 NOTE: this option is for experts only.
 457 EODESCR
 458     },
 459     tablet => {
 460         optional => 1,
 461         type => 'boolean',
 462         default => 1,
 463         description => "Enable/disable the USB tablet device.",
 464         verbose_description => "Enable/disable the USB tablet device. This device is " .
 465             "usually needed to allow absolute mouse positioning with VNC. " .
 466             "Else the mouse runs out of sync with normal VNC clients. " .
 467             "If you're running lots of console-only guests on one host, " .
 468             "you may consider disabling this to save some context switches. " .
 469             "This is turned off by default if you use spice (-vga=qxl).",
 470     },
 471     migrate_speed => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 475         minimum => 0,
 476         default => 0,
 477     },
 478     migrate_downtime => {
 479         optional => 1,
 480         type => 'number',
 481         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 482         minimum => 0,
 483         default => 0.1,
 484     },
 485     cdrom => {
 486         optional => 1,
 487         type => 'string', format => 'pve-qm-ide',
 488         typetext => '<volume>',
 489         description => "This is an alias for option -ide2",
 490     },
 491     cpu => {
 492         optional => 1,
 493         description => "Emulated CPU type.",
 494         type => 'string',
 495         format => $cpu_fmt,
 496     },
 497     parent => get_standard_option('pve-snapshot-name', {
 498         optional => 1,
 499         description => "Parent snapshot name. This is used internally, and should not be modified.",
 500     }),
 501     snaptime => {
 502         optional => 1,
 503         description => "Timestamp for snapshots.",
 504         type => 'integer',
 505         minimum => 0,
 506     },
 507     vmstate => {
 508         optional => 1,
 509         type => 'string', format => 'pve-volume-id',
 510         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 511     },
 512     vmstatestorage => get_standard_option('pve-storage-id', {
 513         description => "Default storage for VM state volumes/files.",
 514         optional => 1,
 515     }),
 516     machine => {
 517         description => "Specific the Qemu machine type.",
 518         type => 'string',
 519         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 520         maxLength => 40,
 521         optional => 1,
 522     },
 523     smbios1 => {
 524         description => "Specify SMBIOS type 1 fields.",
 525         type => 'string', format => 'pve-qm-smbios1',
 526         maxLength => 256,
 527         optional => 1,
 528     },
 529     protection => {
 530         optional => 1,
 531         type => 'boolean',
 532         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 533         default => 0,
 534     },
 535     bios => {
 536         optional => 1,
 537         type => 'string',
 538         enum => [ qw(seabios ovmf) ],
 539         description => "Select BIOS implementation.",
 540         default => 'seabios',
 541     },
 542 };
 543
 544 my $confdesc_cloudinit = {
 545     citype => {
 546         optional => 1,
 547         type => 'string',
 548         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 549         enum => ['configdrive2', 'nocloud'],
 550     },
 551     ciuser => {
 552         optional => 1,
 553         type => 'string',
 554         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 555     },
 556     cipassword => {
 557         optional => 1,
 558         type => 'string',
 559         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 560     },
 561     searchdomain => {
 562         optional => 1,
 563         type => 'string',
 564         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 565     },
 566     nameserver => {
 567         optional => 1,
 568         type => 'string', format => 'address-list',
 569         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 570     },
 571     sshkeys => {
 572         optional => 1,
 573         type => 'string',
 574         format => 'urlencoded',
 575         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 576     },
 577 };
 578
 579 # what about other qemu settings ?
 580 #cpu => 'string',
 581 #machine => 'string',
 582 #fda => 'file',
 583 #fdb => 'file',
 584 #mtdblock => 'file',
 585 #sd => 'file',
 586 #pflash => 'file',
 587 #snapshot => 'bool',
 588 #bootp => 'file',
 589 ##tftp => 'dir',
 590 ##smb => 'dir',
 591 #kernel => 'file',
 592 #append => 'string',
 593 #initrd => 'file',
 594 ##soundhw => 'string',
 595
 596 while (my ($k, $v) = each %$confdesc) {
 597     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 598 }
 599
 600 my $MAX_IDE_DISKS = 4;
 601 my $MAX_SCSI_DISKS = 14;
 602 my $MAX_VIRTIO_DISKS = 16;
 603 my $MAX_SATA_DISKS = 6;
 604 my $MAX_USB_DEVICES = 5;
 605 my $MAX_NETS = 32;
 606 my $MAX_UNUSED_DISKS = 8;
 607 my $MAX_HOSTPCI_DEVICES = 4;
 608 my $MAX_SERIAL_PORTS = 4;
 609 my $MAX_PARALLEL_PORTS = 3;
 610 my $MAX_NUMA = 8;
 611
 612 my $numa_fmt = {
 613     cpus => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "CPUs accessing this NUMA node.",
 617         format_description => "id[-id];...",
 618     },
 619     memory => {
 620         type => "number",
 621         description => "Amount of memory this NUMA node provides.",
 622         optional => 1,
 623     },
 624     hostnodes => {
 625         type => "string",
 626         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 627         description => "Host NUMA nodes to use.",
 628         format_description => "id[-id];...",
 629         optional => 1,
 630     },
 631     policy => {
 632         type => 'string',
 633         enum => [qw(preferred bind interleave)],
 634         description => "NUMA allocation policy.",
 635         optional => 1,
 636     },
 637 };
 638 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 639 my $numadesc = {
 640     optional => 1,
 641     type => 'string', format => $numa_fmt,
 642     description => "NUMA topology.",
 643 };
 644 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 645
 646 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 647     $confdesc->{"numa$i"} = $numadesc;
 648 }
 649
 650 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 651                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 652                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 653 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 654
 655 my $net_fmt_bridge_descr = <<__EOD__;
 656 Bridge to attach the network device to. The Proxmox VE standard bridge
 657 is called 'vmbr0'.
 658
 659 If you do not specify a bridge, we create a kvm user (NATed) network
 660 device, which provides DHCP and DNS services. The following addresses
 661 are used:
 662
 663  10.0.2.2   Gateway
 664  10.0.2.3   DNS Server
 665  10.0.2.4   SMB Server
 666
 667 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 668 __EOD__
 669
 670 my $net_fmt = {
 671     macaddr => {
 672         type => 'string',
 673         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 674         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 675         format_description => "XX:XX:XX:XX:XX:XX",
 676         optional => 1,
 677     },
 678     model => {
 679         type => 'string',
 680         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 681         enum => $nic_model_list,
 682         default_key => 1,
 683     },
 684     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 685     bridge => {
 686         type => 'string',
 687         description => $net_fmt_bridge_descr,
 688         format_description => 'bridge',
 689         optional => 1,
 690     },
 691     queues => {
 692         type => 'integer',
 693         minimum => 0, maximum => 16,
 694         description => 'Number of packet queues to be used on the device.',
 695         optional => 1,
 696     },
 697     rate => {
 698         type => 'number',
 699         minimum => 0,
 700         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 701         optional => 1,
 702     },
 703     tag => {
 704         type => 'integer',
 705         minimum => 1, maximum => 4094,
 706         description => 'VLAN tag to apply to packets on this interface.',
 707         optional => 1,
 708     },
 709     trunks => {
 710         type => 'string',
 711         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 712         description => 'VLAN trunks to pass through this interface.',
 713         format_description => 'vlanid[;vlanid...]',
 714         optional => 1,
 715     },
 716     firewall => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be protected by the firewall.',
 719         optional => 1,
 720     },
 721     link_down => {
 722         type => 'boolean',
 723         description => 'Whether this interface should be disconnected (like pulling the plug).',
 724         optional => 1,
 725     },
 726 };
 727
 728 my $netdesc = {
 729     optional => 1,
 730     type => 'string', format => $net_fmt,
 731     description => "Specify network devices.",
 732 };
 733
 734 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 735
 736 my $ipconfig_fmt = {
 737     ip => {
 738         type => 'string',
 739         format => 'pve-ipv4-config',
 740         format_description => 'IPv4Format/CIDR',
 741         description => 'IPv4 address in CIDR format.',
 742         optional => 1,
 743         default => 'dhcp',
 744     },
 745     gw => {
 746         type => 'string',
 747         format => 'ipv4',
 748         format_description => 'GatewayIPv4',
 749         description => 'Default gateway for IPv4 traffic.',
 750         optional => 1,
 751         requires => 'ip',
 752     },
 753     ip6 => {
 754         type => 'string',
 755         format => 'pve-ipv6-config',
 756         format_description => 'IPv6Format/CIDR',
 757         description => 'IPv6 address in CIDR format.',
 758         optional => 1,
 759         default => 'dhcp',
 760     },
 761     gw6 => {
 762         type => 'string',
 763         format => 'ipv6',
 764         format_description => 'GatewayIPv6',
 765         description => 'Default gateway for IPv6 traffic.',
 766         optional => 1,
 767         requires => 'ip6',
 768     },
 769 };
 770 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 771 my $ipconfigdesc = {
 772     optional => 1,
 773     type => 'string', format => 'pve-qm-ipconfig',
 774     description => <<'EODESCR',
 775 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 776
 777 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 778
 779 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 780 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 781
 782 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 786
 787 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 788     $confdesc->{"net$i"} = $netdesc;
 789     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 790 }
 791
 792 foreach my $key (keys %$confdesc_cloudinit) {
 793     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 797 sub verify_volume_id_or_qm_path {
 798     my ($volid, $noerr) = @_;
 799
 800     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 801         return $volid;
 802     }
 803
 804     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 805     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 806     if ($@) {
 807         return undef if $noerr;
 808         die $@;
 809     }
 810     return $volid;
 811 }
 812
 813 my $drivename_hash;
 814
 815 my %drivedesc_base = (
 816     volume => { alias => 'file' },
 817     file => {
 818         type => 'string',
 819         format => 'pve-volume-id-or-qm-path',
 820         default_key => 1,
 821         format_description => 'volume',
 822         description => "The drive's backing volume.",
 823     },
 824     media => {
 825         type => 'string',
 826         enum => [qw(cdrom disk)],
 827         description => "The drive's media type.",
 828         default => 'disk',
 829         optional => 1
 830     },
 831     cyls => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific cylinder count.",
 834         optional => 1
 835     },
 836     heads => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific head count.",
 839         optional => 1
 840     },
 841     secs => {
 842         type => 'integer',
 843         description => "Force the drive's physical geometry to have a specific sector count.",
 844         optional => 1
 845     },
 846     trans => {
 847         type => 'string',
 848         enum => [qw(none lba auto)],
 849         description => "Force disk geometry bios translation mode.",
 850         optional => 1,
 851     },
 852     snapshot => {
 853         type => 'boolean',
 854         description => "Controls qemu's snapshot mode feature."
 855             . " If activated, changes made to the disk are temporary and will"
 856             . " be discarded when the VM is shutdown.",
 857         optional => 1,
 858     },
 859     cache => {
 860         type => 'string',
 861         enum => [qw(none writethrough writeback unsafe directsync)],
 862         description => "The drive's cache mode",
 863         optional => 1,
 864     },
 865     format => get_standard_option('pve-qm-image-format'),
 866     size => {
 867         type => 'string',
 868         format => 'disk-size',
 869         format_description => 'DiskSize',
 870         description => "Disk size. This is purely informational and has no effect.",
 871         optional => 1,
 872     },
 873     backup => {
 874         type => 'boolean',
 875         description => "Whether the drive should be included when making backups.",
 876         optional => 1,
 877     },
 878     replicate => {
 879         type => 'boolean',
 880         description => 'Whether the drive should considered for replication jobs.',
 881         optional => 1,
 882         default => 1,
 883     },
 884     rerror => {
 885         type => 'string',
 886         enum => [qw(ignore report stop)],
 887         description => 'Read error action.',
 888         optional => 1,
 889     },
 890     werror => {
 891         type => 'string',
 892         enum => [qw(enospc ignore report stop)],
 893         description => 'Write error action.',
 894         optional => 1,
 895     },
 896     aio => {
 897         type => 'string',
 898         enum => [qw(native threads)],
 899         description => 'AIO type to use.',
 900         optional => 1,
 901     },
 902     discard => {
 903         type => 'string',
 904         enum => [qw(ignore on)],
 905         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 906         optional => 1,
 907     },
 908     detect_zeroes => {
 909         type => 'boolean',
 910         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 911         optional => 1,
 912     },
 913     serial => {
 914         type => 'string',
 915         format => 'urlencoded',
 916         format_description => 'serial',
 917         maxLength => 20*3, # *3 since it's %xx url enoded
 918         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 919         optional => 1,
 920     },
 921     shared => {
 922         type => 'boolean',
 923         description => 'Mark this locally-managed volume as available on all nodes',
 924         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 925         optional => 1,
 926         default => 0,
 927     }
 928 );
 929
 930 my %iothread_fmt = ( iothread => {
 931         type => 'boolean',
 932         description => "Whether to use iothreads for this drive",
 933         optional => 1,
 934 });
 935
 936 my %model_fmt = (
 937     model => {
 938         type => 'string',
 939         format => 'urlencoded',
 940         format_description => 'model',
 941         maxLength => 40*3, # *3 since it's %xx url enoded
 942         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 943         optional => 1,
 944     },
 945 );
 946
 947 my %queues_fmt = (
 948     queues => {
 949         type => 'integer',
 950         description => "Number of queues.",
 951         minimum => 2,
 952         optional => 1
 953     }
 954 );
 955
 956 my %scsiblock_fmt = (
 957     scsiblock => {
 958         type => 'boolean',
 959         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 960         optional => 1,
 961         default => 0,
 962     },
 963 );
 964
 965 my $add_throttle_desc = sub {
 966     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 967     my $d = {
 968         type => $type,
 969         format_description => $unit,
 970         description => "Maximum $what in $longunit.",
 971         optional => 1,
 972     };
 973     $d->{minimum} = $minimum if defined($minimum);
 974     $drivedesc_base{$key} = $d;
 975 };
 976 # throughput: (leaky bucket)
 977 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 979 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 980 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 982 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 983 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 984 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 985 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 986
 987 # pools: (pool of IO before throttling starts taking effect)
 988 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 990 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 991 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 992 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 993 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 994
 995 # burst lengths
 996 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 998 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1001 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1002
1003 # legacy support
1004 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1005 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1006 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1007 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1008
1009 my $ide_fmt = {
1010     %drivedesc_base,
1011     %model_fmt,
1012 };
1013 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1014
1015 my $idedesc = {
1016     optional => 1,
1017     type => 'string', format => $ide_fmt,
1018     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1019 };
1020 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1021
1022 my $scsi_fmt = {
1023     %drivedesc_base,
1024     %iothread_fmt,
1025     %queues_fmt,
1026     %scsiblock_fmt,
1027 };
1028 my $scsidesc = {
1029     optional => 1,
1030     type => 'string', format => $scsi_fmt,
1031     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1034
1035 my $sata_fmt = {
1036     %drivedesc_base,
1037 };
1038 my $satadesc = {
1039     optional => 1,
1040     type => 'string', format => $sata_fmt,
1041     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1044
1045 my $virtio_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048 };
1049 my $virtiodesc = {
1050     optional => 1,
1051     type => 'string', format => $virtio_fmt,
1052     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1053 };
1054 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1055
1056 my $alldrive_fmt = {
1057     %drivedesc_base,
1058     %iothread_fmt,
1059     %model_fmt,
1060     %queues_fmt,
1061     %scsiblock_fmt,
1062 };
1063
1064 my $efidisk_fmt = {
1065     volume => { alias => 'file' },
1066     file => {
1067         type => 'string',
1068         format => 'pve-volume-id-or-qm-path',
1069         default_key => 1,
1070         format_description => 'volume',
1071         description => "The drive's backing volume.",
1072     },
1073     format => get_standard_option('pve-qm-image-format'),
1074     size => {
1075         type => 'string',
1076         format => 'disk-size',
1077         format_description => 'DiskSize',
1078         description => "Disk size. This is purely informational and has no effect.",
1079         optional => 1,
1080     },
1081 };
1082
1083 my $efidisk_desc = {
1084     optional => 1,
1085     type => 'string', format => $efidisk_fmt,
1086     description => "Configure a Disk for storing EFI vars",
1087 };
1088
1089 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1090
1091 my $usb_fmt = {
1092     host => {
1093         default_key => 1,
1094         type => 'string', format => 'pve-qm-usb-device',
1095         format_description => 'HOSTUSBDEVICE|spice',
1096         description => <<EODESCR,
1097 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1098
1099  'bus-port(.port)*' (decimal numbers) or
1100  'vendor_id:product_id' (hexadeciaml numbers) or
1101  'spice'
1102
1103 You can use the 'lsusb -t' command to list existing usb devices.
1104
1105 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1106
1107 The value 'spice' can be used to add a usb redirection devices for spice.
1108 EODESCR
1109     },
1110     usb3 => {
1111         optional => 1,
1112         type => 'boolean',
1113         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1114         default => 0,
1115     },
1116 };
1117
1118 my $usbdesc = {
1119     optional => 1,
1120     type => 'string', format => $usb_fmt,
1121     description => "Configure an USB device (n is 0 to 4).",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1124
1125 # NOTE: the match-groups of this regex are used in parse_hostpci
1126 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1127 my $hostpci_fmt = {
1128     host => {
1129         default_key => 1,
1130         type => 'string',
1131         pattern => qr/$PCIRE(;$PCIRE)*/,
1132         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1133         description => <<EODESCR,
1134 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1135 of PCI virtual functions of the host. HOSTPCIID syntax is:
1136
1137 'bus:dev.func' (hexadecimal numbers)
1138
1139 You can us the 'lspci' command to list existing PCI devices.
1140 EODESCR
1141     },
1142     rombar => {
1143         type => 'boolean',
1144         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1145         optional => 1,
1146         default => 1,
1147     },
1148     romfile => {
1149         type => 'string',
1150         pattern => '[^,;]+',
1151         format_description => 'string',
1152         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1153         optional => 1,
1154     },
1155     pcie => {
1156         type => 'boolean',
1157         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1158         optional => 1,
1159         default => 0,
1160     },
1161     'x-vga' => {
1162         type => 'boolean',
1163         description =>  "Enable vfio-vga device support.",
1164         optional => 1,
1165         default => 0,
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1169
1170 my $hostpcidesc = {
1171         optional => 1,
1172         type => 'string', format => 'pve-qm-hostpci',
1173         description => "Map host PCI devices into guest.",
1174         verbose_description =>  <<EODESCR,
1175 Map host PCI devices into guest.
1176
1177 NOTE: This option allows direct access to host hardware. So it is no longer
1178 possible to migrate such machines - use with special care.
1179
1180 CAUTION: Experimental! User reported problems with this option.
1181 EODESCR
1182 };
1183 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1184
1185 my $serialdesc = {
1186         optional => 1,
1187         type => 'string',
1188         pattern => '(/dev/.+|socket)',
1189         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1190         verbose_description =>  <<EODESCR,
1191 Create a serial device inside the VM (n is 0 to 3), and pass through a
1192 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1193 host side (use 'qm terminal' to open a terminal connection).
1194
1195 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1196
1197 CAUTION: Experimental! User reported problems with this option.
1198 EODESCR
1199 };
1200
1201 my $paralleldesc= {
1202         optional => 1,
1203         type => 'string',
1204         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1205         description =>  "Map host parallel devices (n is 0 to 2).",
1206         verbose_description =>  <<EODESCR,
1207 Map host parallel devices (n is 0 to 2).
1208
1209 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1210
1211 CAUTION: Experimental! User reported problems with this option.
1212 EODESCR
1213 };
1214
1215 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1216     $confdesc->{"parallel$i"} = $paralleldesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1220     $confdesc->{"serial$i"} = $serialdesc;
1221 }
1222
1223 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1224     $confdesc->{"hostpci$i"} = $hostpcidesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1228     $drivename_hash->{"ide$i"} = 1;
1229     $confdesc->{"ide$i"} = $idedesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1233     $drivename_hash->{"sata$i"} = 1;
1234     $confdesc->{"sata$i"} = $satadesc;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1238     $drivename_hash->{"scsi$i"} = 1;
1239     $confdesc->{"scsi$i"} = $scsidesc ;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1243     $drivename_hash->{"virtio$i"} = 1;
1244     $confdesc->{"virtio$i"} = $virtiodesc;
1245 }
1246
1247 $drivename_hash->{efidisk0} = 1;
1248 $confdesc->{efidisk0} = $efidisk_desc;
1249
1250 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1251     $confdesc->{"usb$i"} = $usbdesc;
1252 }
1253
1254 my $unuseddesc = {
1255     optional => 1,
1256     type => 'string', format => 'pve-volume-id',
1257     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1258 };
1259
1260 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1261     $confdesc->{"unused$i"} = $unuseddesc;
1262 }
1263
1264 my $kvm_api_version = 0;
1265
1266 sub kvm_version {
1267
1268     return $kvm_api_version if $kvm_api_version;
1269
1270     my $fh = IO::File->new("</dev/kvm") ||
1271         return 0;
1272
1273     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1274         $kvm_api_version = $v;
1275     }
1276
1277     $fh->close();
1278
1279     return  $kvm_api_version;
1280 }
1281
1282 my $kvm_user_version;
1283
1284 sub kvm_user_version {
1285
1286     return $kvm_user_version if $kvm_user_version;
1287
1288     $kvm_user_version = 'unknown';
1289
1290     my $code = sub {
1291         my $line = shift;
1292         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1293             $kvm_user_version = $2;
1294         }
1295     };
1296
1297     eval { run_command("kvm -version", outfunc => $code); };
1298     warn $@ if $@;
1299
1300     return $kvm_user_version;
1301
1302 }
1303
1304 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1305
1306 sub valid_drive_names {
1307     # order is important - used to autoselect boot disk
1308     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1309             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1310             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1311             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1312             'efidisk0');
1313 }
1314
1315 sub is_valid_drivename {
1316     my $dev = shift;
1317
1318     return defined($drivename_hash->{$dev});
1319 }
1320
1321 sub option_exists {
1322     my $key = shift;
1323     return defined($confdesc->{$key});
1324 }
1325
1326 sub nic_models {
1327     return $nic_model_list;
1328 }
1329
1330 sub os_list_description {
1331
1332     return {
1333         other => 'Other',
1334         wxp => 'Windows XP',
1335         w2k => 'Windows 2000',
1336         w2k3 =>, 'Windows 2003',
1337         w2k8 => 'Windows 2008',
1338         wvista => 'Windows Vista',
1339         win7 => 'Windows 7',
1340         win8 => 'Windows 8/2012',
1341         win10 => 'Windows 10/2016',
1342         l24 => 'Linux 2.4',
1343         l26 => 'Linux 2.6',
1344     };
1345 }
1346
1347 my $cdrom_path;
1348
1349 sub get_cdrom_path {
1350
1351     return  $cdrom_path if $cdrom_path;
1352
1353     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1354     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1355     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1356 }
1357
1358 sub get_iso_path {
1359     my ($storecfg, $vmid, $cdrom) = @_;
1360
1361     if ($cdrom eq 'cdrom') {
1362         return get_cdrom_path();
1363     } elsif ($cdrom eq 'none') {
1364         return '';
1365     } elsif ($cdrom =~ m|^/|) {
1366         return $cdrom;
1367     } else {
1368         return PVE::Storage::path($storecfg, $cdrom);
1369     }
1370 }
1371
1372 # try to convert old style file names to volume IDs
1373 sub filename_to_volume_id {
1374     my ($vmid, $file, $media) = @_;
1375
1376      if (!($file eq 'none' || $file eq 'cdrom' ||
1377           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1378
1379         return undef if $file =~ m|/|;
1380
1381         if ($media && $media eq 'cdrom') {
1382             $file = "local:iso/$file";
1383         } else {
1384             $file = "local:$vmid/$file";
1385         }
1386     }
1387
1388     return $file;
1389 }
1390
1391 sub verify_media_type {
1392     my ($opt, $vtype, $media) = @_;
1393
1394     return if !$media;
1395
1396     my $etype;
1397     if ($media eq 'disk') {
1398         $etype = 'images';
1399     } elsif ($media eq 'cdrom') {
1400         $etype = 'iso';
1401     } else {
1402         die "internal error";
1403     }
1404
1405     return if ($vtype eq $etype);
1406
1407     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1408 }
1409
1410 sub cleanup_drive_path {
1411     my ($opt, $storecfg, $drive) = @_;
1412
1413     # try to convert filesystem paths to volume IDs
1414
1415     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1416         ($drive->{file} !~ m|^/dev/.+|) &&
1417         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1418         ($drive->{file} !~ m/^\d+$/)) {
1419         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1420         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1421         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1422         verify_media_type($opt, $vtype, $drive->{media});
1423         $drive->{file} = $volid;
1424     }
1425
1426     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1427 }
1428
1429 sub parse_hotplug_features {
1430     my ($data) = @_;
1431
1432     my $res = {};
1433
1434     return $res if $data eq '0';
1435
1436     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1437
1438     foreach my $feature (PVE::Tools::split_list($data)) {
1439         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1440             $res->{$1} = 1;
1441         } else {
1442             die "invalid hotplug feature '$feature'\n";
1443         }
1444     }
1445     return $res;
1446 }
1447
1448 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1449 sub pve_verify_hotplug_features {
1450     my ($value, $noerr) = @_;
1451
1452     return $value if parse_hotplug_features($value);
1453
1454     return undef if $noerr;
1455
1456     die "unable to parse hotplug option\n";
1457 }
1458
1459 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1460 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1461 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1462 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1463 #        [,iothread=on][,serial=serial][,model=model]
1464
1465 sub parse_drive {
1466     my ($key, $data) = @_;
1467
1468     my ($interface, $index);
1469
1470     if ($key =~ m/^([^\d]+)(\d+)$/) {
1471         $interface = $1;
1472         $index = $2;
1473     } else {
1474         return undef;
1475     }
1476
1477     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1478                                      : $confdesc->{$key}->{format};
1479     if (!$desc) {
1480         warn "invalid drive key: $key\n";
1481         return undef;
1482     }
1483     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1484     return undef if !$res;
1485     $res->{interface} = $interface;
1486     $res->{index} = $index;
1487
1488     my $error = 0;
1489     foreach my $opt (qw(bps bps_rd bps_wr)) {
1490         if (my $bps = defined(delete $res->{$opt})) {
1491             if (defined($res->{"m$opt"})) {
1492                 warn "both $opt and m$opt specified\n";
1493                 ++$error;
1494                 next;
1495             }
1496             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1497         }
1498     }
1499
1500     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1501     for my $requirement (
1502         [mbps_max => 'mbps'],
1503         [mbps_rd_max => 'mbps_rd'],
1504         [mbps_wr_max => 'mbps_wr'],
1505         [miops_max => 'miops'],
1506         [miops_rd_max => 'miops_rd'],
1507         [miops_wr_max => 'miops_wr'],
1508         [bps_max_length => 'mbps_max'],
1509         [bps_rd_max_length => 'mbps_rd_max'],
1510         [bps_wr_max_length => 'mbps_wr_max'],
1511         [iops_max_length => 'iops_max'],
1512         [iops_rd_max_length => 'iops_rd_max'],
1513         [iops_wr_max_length => 'iops_wr_max']) {
1514         my ($option, $requires) = @$requirement;
1515         if ($res->{$option} && !$res->{$requires}) {
1516             warn "$option requires $requires\n";
1517             ++$error;
1518         }
1519     }
1520
1521     return undef if $error;
1522
1523     return undef if $res->{mbps_rd} && $res->{mbps};
1524     return undef if $res->{mbps_wr} && $res->{mbps};
1525     return undef if $res->{iops_rd} && $res->{iops};
1526     return undef if $res->{iops_wr} && $res->{iops};
1527
1528     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1529         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1530         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1531         return undef if $res->{interface} eq 'virtio';
1532     }
1533
1534     if (my $size = $res->{size}) {
1535         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub print_drive {
1542     my ($vmid, $drive) = @_;
1543     my $data = { %$drive };
1544     delete $data->{$_} for qw(index interface);
1545     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1546 }
1547
1548 sub scsi_inquiry {
1549     my($fh, $noerr) = @_;
1550
1551     my $SG_IO = 0x2285;
1552     my $SG_GET_VERSION_NUM = 0x2282;
1553
1554     my $versionbuf = "\x00" x 8;
1555     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1556     if (!$ret) {
1557         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1558         return undef;
1559     }
1560     my $version = unpack("I", $versionbuf);
1561     if ($version < 30000) {
1562         die "scsi generic interface too old\n"  if !$noerr;
1563         return undef;
1564     }
1565
1566     my $buf = "\x00" x 36;
1567     my $sensebuf = "\x00" x 8;
1568     my $cmd = pack("C x3 C x1", 0x12, 36);
1569
1570     # see /usr/include/scsi/sg.h
1571     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1572
1573     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1574                       length($sensebuf), 0, length($buf), $buf,
1575                       $cmd, $sensebuf, 6000);
1576
1577     $ret = ioctl($fh, $SG_IO, $packet);
1578     if (!$ret) {
1579         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1580         return undef;
1581     }
1582
1583     my @res = unpack($sg_io_hdr_t, $packet);
1584     if ($res[17] || $res[18]) {
1585         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1586         return undef;
1587     }
1588
1589     my $res = {};
1590     (my $byte0, my $byte1, $res->{vendor},
1591      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1592
1593     $res->{removable} = $byte1 & 128 ? 1 : 0;
1594     $res->{type} = $byte0 & 31;
1595
1596     return $res;
1597 }
1598
1599 sub path_is_scsi {
1600     my ($path) = @_;
1601
1602     my $fh = IO::File->new("+<$path") || return undef;
1603     my $res = scsi_inquiry($fh, 1);
1604     close($fh);
1605
1606     return $res;
1607 }
1608
1609 sub machine_type_is_q35 {
1610     my ($conf) = @_;
1611
1612     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1613 }
1614
1615 sub print_tabletdevice_full {
1616     my ($conf) = @_;
1617
1618     my $q35 = machine_type_is_q35($conf);
1619
1620     # we use uhci for old VMs because tablet driver was buggy in older qemu
1621     my $usbbus = $q35 ? "ehci" : "uhci";
1622
1623     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1624 }
1625
1626 sub print_drivedevice_full {
1627     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1628
1629     my $device = '';
1630     my $maxdev = 0;
1631
1632     if ($drive->{interface} eq 'virtio') {
1633         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1634         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1635         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1636     } elsif ($drive->{interface} eq 'scsi') {
1637
1638         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1639         my $unit = $drive->{index} % $maxdev;
1640         my $devicetype = 'hd';
1641         my $path = '';
1642         if (drive_is_cdrom($drive)) {
1643             $devicetype = 'cd';
1644         } else {
1645             if ($drive->{file} =~ m|^/|) {
1646                 $path = $drive->{file};
1647                 if (my $info = path_is_scsi($path)) {
1648                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1649                         $devicetype = 'block';
1650                     } elsif ($info->{type} == 1) { # tape
1651                         $devicetype = 'generic';
1652                     }
1653                 }
1654             } else {
1655                  $path = PVE::Storage::path($storecfg, $drive->{file});
1656             }
1657
1658             if($path =~ m/^iscsi\:\/\//){
1659                 $devicetype = 'generic';
1660             }
1661         }
1662
1663         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         } else {
1666             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1667         }
1668
1669     } elsif ($drive->{interface} eq 'ide'){
1670         $maxdev = 2;
1671         my $controller = int($drive->{index} / $maxdev);
1672         my $unit = $drive->{index} % $maxdev;
1673         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1674
1675         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1676         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1677             $model = URI::Escape::uri_unescape($model);
1678             $device .= ",model=$model";
1679         }
1680     } elsif ($drive->{interface} eq 'sata'){
1681         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1682         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1683         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1684     } elsif ($drive->{interface} eq 'usb') {
1685         die "implement me";
1686         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1687     } else {
1688         die "unsupported interface type";
1689     }
1690
1691     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1692
1693     if (my $serial = $drive->{serial}) {
1694         $serial = URI::Escape::uri_unescape($serial);
1695         $device .= ",serial=$serial";
1696     }
1697
1698
1699     return $device;
1700 }
1701
1702 sub get_initiator_name {
1703     my $initiator;
1704
1705     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1706     while (defined(my $line = <$fh>)) {
1707         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1708         $initiator = $1;
1709         last;
1710     }
1711     $fh->close();
1712
1713     return $initiator;
1714 }
1715
1716 sub print_drive_full {
1717     my ($storecfg, $vmid, $drive) = @_;
1718
1719     my $path;
1720     my $volid = $drive->{file};
1721     my $format;
1722
1723     if (drive_is_cdrom($drive)) {
1724         $path = get_iso_path($storecfg, $vmid, $volid);
1725     } else {
1726         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1727         if ($storeid) {
1728             $path = PVE::Storage::path($storecfg, $volid);
1729             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1730             $format = qemu_img_format($scfg, $volname);
1731         } else {
1732             $path = $volid;
1733             $format = "raw";
1734         }
1735    }
1736
1737     my $opts = '';
1738     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1739     foreach my $o (@qemu_drive_options) {
1740         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1741     }
1742
1743     # snapshot only accepts on|off
1744     if (defined($drive->{snapshot})) {
1745         my $v = $drive->{snapshot} ? 'on' : 'off';
1746         $opts .= ",snapshot=$v";
1747     }
1748
1749     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1750         my ($dir, $qmpname) = @$type;
1751         if (my $v = $drive->{"mbps$dir"}) {
1752             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1753         }
1754         if (my $v = $drive->{"mbps${dir}_max"}) {
1755             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1756         }
1757         if (my $v = $drive->{"bps${dir}_max_length"}) {
1758             $opts .= ",throttling.bps$qmpname-max-length=$v";
1759         }
1760         if (my $v = $drive->{"iops${dir}"}) {
1761             $opts .= ",throttling.iops$qmpname=$v";
1762         }
1763         if (my $v = $drive->{"iops${dir}_max"}) {
1764             $opts .= ",throttling.iops$qmpname-max=$v";
1765         }
1766         if (my $v = $drive->{"iops${dir}_max_length"}) {
1767             $opts .= ",throttling.iops$qmpname-max-length=$v";
1768         }
1769     }
1770
1771     $opts .= ",format=$format" if $format && !$drive->{format};
1772
1773     my $cache_direct = 0;
1774
1775     if (my $cache = $drive->{cache}) {
1776         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1777     } elsif (!drive_is_cdrom($drive)) {
1778         $opts .= ",cache=none";
1779         $cache_direct = 1;
1780     }
1781
1782     # aio native works only with O_DIRECT
1783     if (!$drive->{aio}) {
1784         if($cache_direct) {
1785             $opts .= ",aio=native";
1786         } else {
1787             $opts .= ",aio=threads";
1788         }
1789     }
1790
1791     if (!drive_is_cdrom($drive)) {
1792         my $detectzeroes;
1793         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1794             $detectzeroes = 'off';
1795         } elsif ($drive->{discard}) {
1796             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1797         } else {
1798             # This used to be our default with discard not being specified:
1799             $detectzeroes = 'on';
1800         }
1801         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1802     }
1803
1804     my $pathinfo = $path ? "file=$path," : '';
1805
1806     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1807 }
1808
1809 sub print_netdevice_full {
1810     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1811
1812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1813
1814     my $device = $net->{model};
1815     if ($net->{model} eq 'virtio') {
1816          $device = 'virtio-net-pci';
1817      };
1818
1819     my $pciaddr = print_pci_addr("$netid", $bridges);
1820     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1821     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1822         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1823         my $vectors = $net->{queues} * 2 + 2;
1824         $tmpstr .= ",vectors=$vectors,mq=on";
1825     }
1826     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1827
1828     if ($use_old_bios_files) {
1829         my $romfile;
1830         if ($device eq 'virtio-net-pci') {
1831             $romfile = 'pxe-virtio.rom';
1832         } elsif ($device eq 'e1000') {
1833             $romfile = 'pxe-e1000.rom';
1834         } elsif ($device eq 'ne2k') {
1835             $romfile = 'pxe-ne2k_pci.rom';
1836         } elsif ($device eq 'pcnet') {
1837             $romfile = 'pxe-pcnet.rom';
1838         } elsif ($device eq 'rtl8139') {
1839             $romfile = 'pxe-rtl8139.rom';
1840         }
1841         $tmpstr .= ",romfile=$romfile" if $romfile;
1842     }
1843
1844     return $tmpstr;
1845 }
1846
1847 sub print_netdev_full {
1848     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1849
1850     my $i = '';
1851     if ($netid =~ m/^net(\d+)$/) {
1852         $i = int($1);
1853     }
1854
1855     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1856
1857     my $ifname = "tap${vmid}i$i";
1858
1859     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1860     die "interface name '$ifname' is too long (max 15 character)\n"
1861         if length($ifname) >= 16;
1862
1863     my $vhostparam = '';
1864     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1865
1866     my $vmname = $conf->{name} || "vm$vmid";
1867
1868     my $netdev = "";
1869     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1870
1871     if ($net->{bridge}) {
1872         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1873     } else {
1874         $netdev = "type=user,id=$netid,hostname=$vmname";
1875     }
1876
1877     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1878
1879     return $netdev;
1880 }
1881
1882
1883 sub print_cpu_device {
1884     my ($conf, $id) = @_;
1885
1886     my $kvm = $conf->{kvm} // 1;
1887     my $cpu = $kvm ? "kvm64" : "qemu64";
1888     if (my $cputype = $conf->{cpu}) {
1889         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1890             or die "Cannot parse cpu description: $cputype\n";
1891         $cpu = $cpuconf->{cputype};
1892     }
1893
1894     my $cores = $conf->{cores} || 1;
1895
1896     my $current_core = ($id - 1) % $cores;
1897     my $current_socket = int(($id - 1 - $current_core)/$cores);
1898
1899     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1900 }
1901
1902 sub drive_is_cloudinit {
1903     my ($drive) = @_;
1904     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1905 }
1906
1907 sub drive_is_cdrom {
1908     my ($drive, $exclude_cloudinit) = @_;
1909
1910     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1911
1912     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1913
1914 }
1915
1916 sub parse_number_sets {
1917     my ($set) = @_;
1918     my $res = [];
1919     foreach my $part (split(/;/, $set)) {
1920         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1921             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1922             push @$res, [ $1, $2 ];
1923         } else {
1924             die "invalid range: $part\n";
1925         }
1926     }
1927     return $res;
1928 }
1929
1930 sub parse_numa {
1931     my ($data) = @_;
1932
1933     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1934     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1935     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1936     return $res;
1937 }
1938
1939 sub parse_hostpci {
1940     my ($value) = @_;
1941
1942     return undef if !$value;
1943
1944     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1945
1946     my @idlist = split(/;/, $res->{host});
1947     delete $res->{host};
1948     foreach my $id (@idlist) {
1949         if ($id =~ /^$PCIRE$/) {
1950             if (defined($2)) {
1951                 push @{$res->{pciid}}, { id => $1, function => $2 };
1952             } else {
1953                 my $pcidevices = lspci($1);
1954                 $res->{pciid} = $pcidevices->{$1};
1955             }
1956         } else {
1957             # should have been caught by parse_property_string already
1958             die "failed to parse PCI id: $id\n";
1959         }
1960     }
1961     return $res;
1962 }
1963
1964 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1965 sub parse_net {
1966     my ($data) = @_;
1967
1968     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1969     if ($@) {
1970         warn $@;
1971         return undef;
1972     }
1973     if (!defined($res->{macaddr})) {
1974         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1975         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1976     }
1977     return $res;
1978 }
1979
1980 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1981 sub parse_ipconfig {
1982     my ($data) = @_;
1983
1984     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return undef;
1988     }
1989
1990     if ($res->{gw} && !$res->{ip}) {
1991         warn 'gateway specified without specifying an IP address';
1992         return undef;
1993     }
1994     if ($res->{gw6} && !$res->{ip6}) {
1995         warn 'IPv6 gateway specified without specifying an IPv6 address';
1996         return undef;
1997     }
1998     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1999         warn 'gateway specified together with DHCP';
2000         return undef;
2001     }
2002     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2003         # gw6 + auto/dhcp
2004         warn "IPv6 gateway specified together with $res->{ip6} address";
2005         return undef;
2006     }
2007
2008     if (!$res->{ip} && !$res->{ip6}) {
2009         return { ip => 'dhcp', ip6 => 'dhcp' };
2010     }
2011
2012     return $res;
2013 }
2014
2015 sub print_net {
2016     my $net = shift;
2017
2018     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2019 }
2020
2021 sub add_random_macs {
2022     my ($settings) = @_;
2023
2024     foreach my $opt (keys %$settings) {
2025         next if $opt !~ m/^net(\d+)$/;
2026         my $net = parse_net($settings->{$opt});
2027         next if !$net;
2028         $settings->{$opt} = print_net($net);
2029     }
2030 }
2031
2032 sub vm_is_volid_owner {
2033     my ($storecfg, $vmid, $volid) = @_;
2034
2035     if ($volid !~  m|^/|) {
2036         my ($path, $owner);
2037         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2038         if ($owner && ($owner == $vmid)) {
2039             return 1;
2040         }
2041     }
2042
2043     return undef;
2044 }
2045
2046 sub split_flagged_list {
2047     my $text = shift || '';
2048     $text =~ s/[,;]/ /g;
2049     $text =~ s/^\s+//;
2050     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2051 }
2052
2053 sub join_flagged_list {
2054     my ($how, $lst) = @_;
2055     join $how, map { $lst->{$_} . $_ } keys %$lst;
2056 }
2057
2058 sub vmconfig_delete_pending_option {
2059     my ($conf, $key, $force) = @_;
2060
2061     delete $conf->{pending}->{$key};
2062     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2063     $pending_delete_hash->{$key} = $force ? '!' : '';
2064     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2065 }
2066
2067 sub vmconfig_undelete_pending_option {
2068     my ($conf, $key) = @_;
2069
2070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2071     delete $pending_delete_hash->{$key};
2072
2073     if (%$pending_delete_hash) {
2074         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2075     } else {
2076         delete $conf->{pending}->{delete};
2077     }
2078 }
2079
2080 sub vmconfig_register_unused_drive {
2081     my ($storecfg, $vmid, $conf, $drive) = @_;
2082
2083     if (drive_is_cloudinit($drive)) {
2084         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2085         warn $@ if $@;
2086     } elsif (!drive_is_cdrom($drive)) {
2087         my $volid = $drive->{file};
2088         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2089             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2090         }
2091     }
2092 }
2093
2094 sub vmconfig_cleanup_pending {
2095     my ($conf) = @_;
2096
2097     # remove pending changes when nothing changed
2098     my $changes;
2099     foreach my $opt (keys %{$conf->{pending}}) {
2100         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2101             $changes = 1;
2102             delete $conf->{pending}->{$opt};
2103         }
2104     }
2105
2106     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2107     my $pending_delete_hash = {};
2108     while (my ($opt, $force) = each %$current_delete_hash) {
2109         if (defined($conf->{$opt})) {
2110             $pending_delete_hash->{$opt} = $force;
2111         } else {
2112             $changes = 1;
2113         }
2114     }
2115
2116     if (%$pending_delete_hash) {
2117         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2118     } else {
2119         delete $conf->{pending}->{delete};
2120     }
2121
2122     return $changes;
2123 }
2124
2125 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2126 my $smbios1_fmt = {
2127     uuid => {
2128         type => 'string',
2129         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2130         format_description => 'UUID',
2131         description => "Set SMBIOS1 UUID.",
2132         optional => 1,
2133     },
2134     version => {
2135         type => 'string',
2136         pattern => '\S+',
2137         format_description => 'string',
2138         description => "Set SMBIOS1 version.",
2139         optional => 1,
2140     },
2141     serial => {
2142         type => 'string',
2143         pattern => '\S+',
2144         format_description => 'string',
2145         description => "Set SMBIOS1 serial number.",
2146         optional => 1,
2147     },
2148     manufacturer => {
2149         type => 'string',
2150         pattern => '\S+',
2151         format_description => 'string',
2152         description => "Set SMBIOS1 manufacturer.",
2153         optional => 1,
2154     },
2155     product => {
2156         type => 'string',
2157         pattern => '\S+',
2158         format_description => 'string',
2159         description => "Set SMBIOS1 product ID.",
2160         optional => 1,
2161     },
2162     sku => {
2163         type => 'string',
2164         pattern => '\S+',
2165         format_description => 'string',
2166         description => "Set SMBIOS1 SKU string.",
2167         optional => 1,
2168     },
2169     family => {
2170         type => 'string',
2171         pattern => '\S+',
2172         format_description => 'string',
2173         description => "Set SMBIOS1 family string.",
2174         optional => 1,
2175     },
2176 };
2177
2178 sub parse_smbios1 {
2179     my ($data) = @_;
2180
2181     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub print_smbios1 {
2187     my ($smbios1) = @_;
2188     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2192
2193 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2194 sub verify_bootdisk {
2195     my ($value, $noerr) = @_;
2196
2197     return $value if is_valid_drivename($value);
2198
2199     return undef if $noerr;
2200
2201     die "invalid boot disk '$value'\n";
2202 }
2203
2204 sub parse_watchdog {
2205     my ($value) = @_;
2206
2207     return undef if !$value;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2210     warn $@ if $@;
2211     return $res;
2212 }
2213
2214 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2215 sub verify_usb_device {
2216     my ($value, $noerr) = @_;
2217
2218     return $value if parse_usb_device($value);
2219
2220     return undef if $noerr;
2221
2222     die "unable to parse usb device\n";
2223 }
2224
2225 # add JSON properties for create and set function
2226 sub json_config_properties {
2227     my $prop = shift;
2228
2229     foreach my $opt (keys %$confdesc) {
2230         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2231         $prop->{$opt} = $confdesc->{$opt};
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # return copy of $confdesc_cloudinit to generate documentation
2238 sub cloudinit_config_properties {
2239
2240     return dclone($confdesc_cloudinit);
2241 }
2242
2243 sub check_type {
2244     my ($key, $value) = @_;
2245
2246     die "unknown setting '$key'\n" if !$confdesc->{$key};
2247
2248     my $type = $confdesc->{$key}->{type};
2249
2250     if (!defined($value)) {
2251         die "got undefined value\n";
2252     }
2253
2254     if ($value =~ m/[\n\r]/) {
2255         die "property contains a line feed\n";
2256     }
2257
2258     if ($type eq 'boolean') {
2259         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2260         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2261         die "type check ('boolean') failed - got '$value'\n";
2262     } elsif ($type eq 'integer') {
2263         return int($1) if $value =~ m/^(\d+)$/;
2264         die "type check ('integer') failed - got '$value'\n";
2265     } elsif ($type eq 'number') {
2266         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2267         die "type check ('number') failed - got '$value'\n";
2268     } elsif ($type eq 'string') {
2269         if (my $fmt = $confdesc->{$key}->{format}) {
2270             PVE::JSONSchema::check_format($fmt, $value);
2271             return $value;
2272         }
2273         $value =~ s/^\"(.*)\"$/$1/;
2274         return $value;
2275     } else {
2276         die "internal error"
2277     }
2278 }
2279
2280 sub check_iommu_support{
2281     #fixme : need to check IOMMU support
2282     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2283
2284     my $iommu=1;
2285     return $iommu;
2286
2287 }
2288
2289 sub touch_config {
2290     my ($vmid) = @_;
2291
2292     my $conf = PVE::QemuConfig->config_file($vmid);
2293     utime undef, undef, $conf;
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2298
2299     my $conffile = PVE::QemuConfig->config_file($vmid);
2300
2301     my $conf = PVE::QemuConfig->load_config($vmid);
2302
2303     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2304
2305     if ($conf->{template}) {
2306         # check if any base image is still used by a linked clone
2307         foreach_drive($conf, sub {
2308                 my ($ds, $drive) = @_;
2309
2310                 return if drive_is_cdrom($drive);
2311
2312                 my $volid = $drive->{file};
2313
2314                 return if !$volid || $volid =~ m|^/|;
2315
2316                 die "base volume '$volid' is still in use by linked cloned\n"
2317                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2318
2319         });
2320     }
2321
2322     # only remove disks owned by this VM
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         return if drive_is_cdrom($drive, 1);
2327
2328         my $volid = $drive->{file};
2329
2330         return if !$volid || $volid =~ m|^/|;
2331
2332         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2333         return if !$path || !$owner || ($owner != $vmid);
2334
2335         eval {
2336             PVE::Storage::vdisk_free($storecfg, $volid);
2337         };
2338         warn "Could not remove disk '$volid', check manually: $@" if $@;
2339
2340     });
2341
2342     if ($keep_empty_config) {
2343         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2344     } else {
2345         unlink $conffile;
2346     }
2347
2348     # also remove unused disk
2349     eval {
2350         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2351
2352         eval {
2353             PVE::Storage::foreach_volid($dl, sub {
2354                 my ($volid, $sid, $volname, $d) = @_;
2355                 PVE::Storage::vdisk_free($storecfg, $volid);
2356             });
2357         };
2358         warn $@ if $@;
2359
2360     };
2361     warn $@ if $@;
2362 }
2363
2364 sub parse_vm_config {
2365     my ($filename, $raw) = @_;
2366
2367     return undef if !defined($raw);
2368
2369     my $res = {
2370         digest => Digest::SHA::sha1_hex($raw),
2371         snapshots => {},
2372         pending => {},
2373     };
2374
2375     $filename =~ m|/qemu-server/(\d+)\.conf$|
2376         || die "got strange filename '$filename'";
2377
2378     my $vmid = $1;
2379
2380     my $conf = $res;
2381     my $descr;
2382     my $section = '';
2383
2384     my @lines = split(/\n/, $raw);
2385     foreach my $line (@lines) {
2386         next if $line =~ m/^\s*$/;
2387
2388         if ($line =~ m/^\[PENDING\]\s*$/i) {
2389             $section = 'pending';
2390             if (defined($descr)) {
2391                 $descr =~ s/\s+$//;
2392                 $conf->{description} = $descr;
2393             }
2394             $descr = undef;
2395             $conf = $res->{$section} = {};
2396             next;
2397
2398         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2399             $section = $1;
2400             if (defined($descr)) {
2401                 $descr =~ s/\s+$//;
2402                 $conf->{description} = $descr;
2403             }
2404             $descr = undef;
2405             $conf = $res->{snapshots}->{$section} = {};
2406             next;
2407         }
2408
2409         if ($line =~ m/^\#(.*)\s*$/) {
2410             $descr = '' if !defined($descr);
2411             $descr .= PVE::Tools::decode_text($1) . "\n";
2412             next;
2413         }
2414
2415         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2416             $descr = '' if !defined($descr);
2417             $descr .= PVE::Tools::decode_text($2);
2418         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2419             $conf->{snapstate} = $1;
2420         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2421             my $key = $1;
2422             my $value = $2;
2423             $conf->{$key} = $value;
2424         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2425             my $value = $1;
2426             if ($section eq 'pending') {
2427                 $conf->{delete} = $value; # we parse this later
2428             } else {
2429                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2430             }
2431         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2432             my $key = $1;
2433             my $value = $2;
2434             eval { $value = check_type($key, $value); };
2435             if ($@) {
2436                 warn "vm $vmid - unable to parse value of '$key' - $@";
2437             } else {
2438                 $key = 'ide2' if $key eq 'cdrom';
2439                 my $fmt = $confdesc->{$key}->{format};
2440                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2441                     my $v = parse_drive($key, $value);
2442                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2443                         $v->{file} = $volid;
2444                         $value = print_drive($vmid, $v);
2445                     } else {
2446                         warn "vm $vmid - unable to parse value of '$key'\n";
2447                         next;
2448                     }
2449                 }
2450
2451                 $conf->{$key} = $value;
2452             }
2453         }
2454     }
2455
2456     if (defined($descr)) {
2457         $descr =~ s/\s+$//;
2458         $conf->{description} = $descr;
2459     }
2460     delete $res->{snapstate}; # just to be sure
2461
2462     return $res;
2463 }
2464
2465 sub write_vm_config {
2466     my ($filename, $conf) = @_;
2467
2468     delete $conf->{snapstate}; # just to be sure
2469
2470     if ($conf->{cdrom}) {
2471         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2472         $conf->{ide2} = $conf->{cdrom};
2473         delete $conf->{cdrom};
2474     }
2475
2476     # we do not use 'smp' any longer
2477     if ($conf->{sockets}) {
2478         delete $conf->{smp};
2479     } elsif ($conf->{smp}) {
2480         $conf->{sockets} = $conf->{smp};
2481         delete $conf->{cores};
2482         delete $conf->{smp};
2483     }
2484
2485     my $used_volids = {};
2486
2487     my $cleanup_config = sub {
2488         my ($cref, $pending, $snapname) = @_;
2489
2490         foreach my $key (keys %$cref) {
2491             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2492                 $key eq 'snapstate' || $key eq 'pending';
2493             my $value = $cref->{$key};
2494             if ($key eq 'delete') {
2495                 die "propertry 'delete' is only allowed in [PENDING]\n"
2496                     if !$pending;
2497                 # fixme: check syntax?
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             die "unable to parse value of '$key' - $@" if $@;
2502
2503             $cref->{$key} = $value;
2504
2505             if (!$snapname && is_valid_drivename($key)) {
2506                 my $drive = parse_drive($key, $value);
2507                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2508             }
2509         }
2510     };
2511
2512     &$cleanup_config($conf);
2513
2514     &$cleanup_config($conf->{pending}, 1);
2515
2516     foreach my $snapname (keys %{$conf->{snapshots}}) {
2517         die "internal error" if $snapname eq 'pending';
2518         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2519     }
2520
2521     # remove 'unusedX' settings if we re-add a volume
2522     foreach my $key (keys %$conf) {
2523         my $value = $conf->{$key};
2524         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2525             delete $conf->{$key};
2526         }
2527     }
2528
2529     my $generate_raw_config = sub {
2530         my ($conf, $pending) = @_;
2531
2532         my $raw = '';
2533
2534         # add description as comment to top of file
2535         if (defined(my $descr = $conf->{description})) {
2536             if ($descr) {
2537                 foreach my $cl (split(/\n/, $descr)) {
2538                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2539                 }
2540             } else {
2541                 $raw .= "#\n" if $pending;
2542             }
2543         }
2544
2545         foreach my $key (sort keys %$conf) {
2546             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2547             $raw .= "$key: $conf->{$key}\n";
2548         }
2549         return $raw;
2550     };
2551
2552     my $raw = &$generate_raw_config($conf);
2553
2554     if (scalar(keys %{$conf->{pending}})){
2555         $raw .= "\n[PENDING]\n";
2556         $raw .= &$generate_raw_config($conf->{pending}, 1);
2557     }
2558
2559     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2560         $raw .= "\n[$snapname]\n";
2561         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2562     }
2563
2564     return $raw;
2565 }
2566
2567 sub load_defaults {
2568
2569     my $res = {};
2570
2571     # we use static defaults from our JSON schema configuration
2572     foreach my $key (keys %$confdesc) {
2573         if (defined(my $default = $confdesc->{$key}->{default})) {
2574             $res->{$key} = $default;
2575         }
2576     }
2577
2578     return $res;
2579 }
2580
2581 sub config_list {
2582     my $vmlist = PVE::Cluster::get_vmlist();
2583     my $res = {};
2584     return $res if !$vmlist || !$vmlist->{ids};
2585     my $ids = $vmlist->{ids};
2586
2587     foreach my $vmid (keys %$ids) {
2588         my $d = $ids->{$vmid};
2589         next if !$d->{node} || $d->{node} ne $nodename;
2590         next if !$d->{type} || $d->{type} ne 'qemu';
2591         $res->{$vmid}->{exists} = 1;
2592     }
2593     return $res;
2594 }
2595
2596 # test if VM uses local resources (to prevent migration)
2597 sub check_local_resources {
2598     my ($conf, $noerr) = @_;
2599
2600     my $loc_res = 0;
2601
2602     $loc_res = 1 if $conf->{hostusb}; # old syntax
2603     $loc_res = 1 if $conf->{hostpci}; # old syntax
2604
2605     foreach my $k (keys %$conf) {
2606         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2607         # sockets are safe: they will recreated be on the target side post-migrate
2608         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2609         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2610     }
2611
2612     die "VM uses local resources\n" if $loc_res && !$noerr;
2613
2614     return $loc_res;
2615 }
2616
2617 # check if used storages are available on all nodes (use by migrate)
2618 sub check_storage_availability {
2619     my ($storecfg, $conf, $node) = @_;
2620
2621     foreach_drive($conf, sub {
2622         my ($ds, $drive) = @_;
2623
2624         my $volid = $drive->{file};
2625         return if !$volid;
2626
2627         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2628         return if !$sid;
2629
2630         # check if storage is available on both nodes
2631         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2632         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2633    });
2634 }
2635
2636 # list nodes where all VM images are available (used by has_feature API)
2637 sub shared_nodes {
2638     my ($conf, $storecfg) = @_;
2639
2640     my $nodelist = PVE::Cluster::get_nodelist();
2641     my $nodehash = { map { $_ => 1 } @$nodelist };
2642     my $nodename = PVE::INotify::nodename();
2643
2644     foreach_drive($conf, sub {
2645         my ($ds, $drive) = @_;
2646
2647         my $volid = $drive->{file};
2648         return if !$volid;
2649
2650         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2651         if ($storeid) {
2652             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2653             if ($scfg->{disable}) {
2654                 $nodehash = {};
2655             } elsif (my $avail = $scfg->{nodes}) {
2656                 foreach my $node (keys %$nodehash) {
2657                     delete $nodehash->{$node} if !$avail->{$node};
2658                 }
2659             } elsif (!$scfg->{shared}) {
2660                 foreach my $node (keys %$nodehash) {
2661                     delete $nodehash->{$node} if $node ne $nodename
2662                 }
2663             }
2664         }
2665     });
2666
2667     return $nodehash
2668 }
2669
2670 sub check_cmdline {
2671     my ($pidfile, $pid) = @_;
2672
2673     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2674     if (defined($fh)) {
2675         my $line = <$fh>;
2676         $fh->close;
2677         return undef if !$line;
2678         my @param = split(/\0/, $line);
2679
2680         my $cmd = $param[0];
2681         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2682
2683         for (my $i = 0; $i < scalar (@param); $i++) {
2684             my $p = $param[$i];
2685             next if !$p;
2686             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2687                 my $p = $param[$i+1];
2688                 return 1 if $p && ($p eq $pidfile);
2689                 return undef;
2690             }
2691         }
2692     }
2693     return undef;
2694 }
2695
2696 sub check_running {
2697     my ($vmid, $nocheck, $node) = @_;
2698
2699     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2700
2701     die "unable to find configuration file for VM $vmid - no such machine\n"
2702         if !$nocheck && ! -f $filename;
2703
2704     my $pidfile = pidfile_name($vmid);
2705
2706     if (my $fd = IO::File->new("<$pidfile")) {
2707         my $st = stat($fd);
2708         my $line = <$fd>;
2709         close($fd);
2710
2711         my $mtime = $st->mtime;
2712         if ($mtime > time()) {
2713             warn "file '$filename' modified in future\n";
2714         }
2715
2716         if ($line =~ m/^(\d+)$/) {
2717             my $pid = $1;
2718             if (check_cmdline($pidfile, $pid)) {
2719                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2720                     return $pid;
2721                 }
2722             }
2723         }
2724     }
2725
2726     return undef;
2727 }
2728
2729 sub vzlist {
2730
2731     my $vzlist = config_list();
2732
2733     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2734
2735     while (defined(my $de = $fd->read)) {
2736         next if $de !~ m/^(\d+)\.pid$/;
2737         my $vmid = $1;
2738         next if !defined($vzlist->{$vmid});
2739         if (my $pid = check_running($vmid)) {
2740             $vzlist->{$vmid}->{pid} = $pid;
2741         }
2742     }
2743
2744     return $vzlist;
2745 }
2746
2747 sub disksize {
2748     my ($storecfg, $conf) = @_;
2749
2750     my $bootdisk = $conf->{bootdisk};
2751     return undef if !$bootdisk;
2752     return undef if !is_valid_drivename($bootdisk);
2753
2754     return undef if !$conf->{$bootdisk};
2755
2756     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2757     return undef if !defined($drive);
2758
2759     return undef if drive_is_cdrom($drive);
2760
2761     my $volid = $drive->{file};
2762     return undef if !$volid;
2763
2764     return $drive->{size};
2765 }
2766
2767 our $vmstatus_return_properties = {
2768     vmid => get_standard_option('pve-vmid'),
2769     status => {
2770         description => "Qemu process status.",
2771         type => 'string',
2772         enum => ['stopped', 'running'],
2773     },
2774     maxmem => {
2775         description => "Maximum memory in bytes.",
2776         type => 'integer',
2777         optional => 1,
2778         renderer => 'bytes',
2779     },
2780     maxdisk => {
2781         description => "Root disk size in bytes.",
2782         type => 'integer',
2783         optional => 1,
2784         renderer => 'bytes',
2785     },
2786     name => {
2787         description => "VM name.",
2788         type => 'string',
2789         optional => 1,
2790     },
2791     qmpstatus => {
2792         description => "Qemu QMP agent status.",
2793         type => 'string',
2794         optional => 1,
2795     },
2796     pid => {
2797         description => "PID of running qemu process.",
2798         type => 'integer',
2799         optional => 1,
2800     },
2801     uptime => {
2802         description => "Uptime.",
2803         type => 'integer',
2804         optional => 1,
2805         renderer => 'duration',
2806     },
2807     cpus => {
2808         description => "Maximum usable CPUs.",
2809         type => 'number',
2810         optional => 1,
2811     },
2812 };
2813
2814 my $last_proc_pid_stat;
2815
2816 # get VM status information
2817 # This must be fast and should not block ($full == false)
2818 # We only query KVM using QMP if $full == true (this can be slow)
2819 sub vmstatus {
2820     my ($opt_vmid, $full) = @_;
2821
2822     my $res = {};
2823
2824     my $storecfg = PVE::Storage::config();
2825
2826     my $list = vzlist();
2827     my $defaults = load_defaults();
2828
2829     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2830
2831     my $cpucount = $cpuinfo->{cpus} || 1;
2832
2833     foreach my $vmid (keys %$list) {
2834         next if $opt_vmid && ($vmid ne $opt_vmid);
2835
2836         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2837         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2838
2839         my $d = { vmid => $vmid };
2840         $d->{pid} = $list->{$vmid}->{pid};
2841
2842         # fixme: better status?
2843         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2844
2845         my $size = disksize($storecfg, $conf);
2846         if (defined($size)) {
2847             $d->{disk} = 0; # no info available
2848             $d->{maxdisk} = $size;
2849         } else {
2850             $d->{disk} = 0;
2851             $d->{maxdisk} = 0;
2852         }
2853
2854         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2855             * ($conf->{cores} || $defaults->{cores});
2856         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2857         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2858
2859         $d->{name} = $conf->{name} || "VM $vmid";
2860         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2861             : $defaults->{memory}*(1024*1024);
2862
2863         if ($conf->{balloon}) {
2864             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2865             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2866                 : $defaults->{shares};
2867         }
2868
2869         $d->{uptime} = 0;
2870         $d->{cpu} = 0;
2871         $d->{mem} = 0;
2872
2873         $d->{netout} = 0;
2874         $d->{netin} = 0;
2875
2876         $d->{diskread} = 0;
2877         $d->{diskwrite} = 0;
2878
2879         $d->{template} = PVE::QemuConfig->is_template($conf);
2880
2881         $d->{serial} = 1 if conf_has_serial($conf);
2882
2883         $res->{$vmid} = $d;
2884     }
2885
2886     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2887     foreach my $dev (keys %$netdev) {
2888         next if $dev !~ m/^tap([1-9]\d*)i/;
2889         my $vmid = $1;
2890         my $d = $res->{$vmid};
2891         next if !$d;
2892
2893         $d->{netout} += $netdev->{$dev}->{receive};
2894         $d->{netin} += $netdev->{$dev}->{transmit};
2895
2896         if ($full) {
2897             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2898             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2899         }
2900
2901     }
2902
2903     my $ctime = gettimeofday;
2904
2905     foreach my $vmid (keys %$list) {
2906
2907         my $d = $res->{$vmid};
2908         my $pid = $d->{pid};
2909         next if !$pid;
2910
2911         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2912         next if !$pstat; # not running
2913
2914         my $used = $pstat->{utime} + $pstat->{stime};
2915
2916         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2917
2918         if ($pstat->{vsize}) {
2919             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2920         }
2921
2922         my $old = $last_proc_pid_stat->{$pid};
2923         if (!$old) {
2924             $last_proc_pid_stat->{$pid} = {
2925                 time => $ctime,
2926                 used => $used,
2927                 cpu => 0,
2928             };
2929             next;
2930         }
2931
2932         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2933
2934         if ($dtime > 1000) {
2935             my $dutime = $used -  $old->{used};
2936
2937             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2938             $last_proc_pid_stat->{$pid} = {
2939                 time => $ctime,
2940                 used => $used,
2941                 cpu => $d->{cpu},
2942             };
2943         } else {
2944             $d->{cpu} = $old->{cpu};
2945         }
2946     }
2947
2948     return $res if !$full;
2949
2950     my $qmpclient = PVE::QMPClient->new();
2951
2952     my $ballooncb = sub {
2953         my ($vmid, $resp) = @_;
2954
2955         my $info = $resp->{'return'};
2956         return if !$info->{max_mem};
2957
2958         my $d = $res->{$vmid};
2959
2960         # use memory assigned to VM
2961         $d->{maxmem} = $info->{max_mem};
2962         $d->{balloon} = $info->{actual};
2963
2964         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2965             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2966             $d->{freemem} = $info->{free_mem};
2967         }
2968
2969         $d->{ballooninfo} = $info;
2970     };
2971
2972     my $blockstatscb = sub {
2973         my ($vmid, $resp) = @_;
2974         my $data = $resp->{'return'} || [];
2975         my $totalrdbytes = 0;
2976         my $totalwrbytes = 0;
2977
2978         for my $blockstat (@$data) {
2979             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2980             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2981
2982             $blockstat->{device} =~ s/drive-//;
2983             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2984         }
2985         $res->{$vmid}->{diskread} = $totalrdbytes;
2986         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2987     };
2988
2989     my $statuscb = sub {
2990         my ($vmid, $resp) = @_;
2991
2992         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2993         # this fails if ballon driver is not loaded, so this must be
2994         # the last commnand (following command are aborted if this fails).
2995         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2996
2997         my $status = 'unknown';
2998         if (!defined($status = $resp->{'return'}->{status})) {
2999             warn "unable to get VM status\n";
3000             return;
3001         }
3002
3003         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3004     };
3005
3006     foreach my $vmid (keys %$list) {
3007         next if $opt_vmid && ($vmid ne $opt_vmid);
3008         next if !$res->{$vmid}->{pid}; # not running
3009         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3010     }
3011
3012     $qmpclient->queue_execute(undef, 2);
3013
3014     foreach my $vmid (keys %$list) {
3015         next if $opt_vmid && ($vmid ne $opt_vmid);
3016         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3017     }
3018
3019     return $res;
3020 }
3021
3022 sub foreach_drive {
3023     my ($conf, $func, @param) = @_;
3024
3025     foreach my $ds (valid_drive_names()) {
3026         next if !defined($conf->{$ds});
3027
3028         my $drive = parse_drive($ds, $conf->{$ds});
3029         next if !$drive;
3030
3031         &$func($ds, $drive, @param);
3032     }
3033 }
3034
3035 sub foreach_volid {
3036     my ($conf, $func, @param) = @_;
3037
3038     my $volhash = {};
3039
3040     my $test_volid = sub {
3041         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
3042
3043         return if !$volid;
3044
3045         $volhash->{$volid}->{cdrom} //= 1;
3046         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3047
3048         $volhash->{$volid}->{replicate} //= 0;
3049         $volhash->{$volid}->{replicate} = 1 if $replicate;
3050
3051         $volhash->{$volid}->{shared} //= 0;
3052         $volhash->{$volid}->{shared} = 1 if $shared;
3053
3054         $volhash->{$volid}->{referenced_in_config} //= 0;
3055         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3056
3057         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3058             if defined($snapname);
3059     };
3060
3061     foreach_drive($conf, sub {
3062         my ($ds, $drive) = @_;
3063         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3064     });
3065
3066     foreach my $snapname (keys %{$conf->{snapshots}}) {
3067         my $snap = $conf->{snapshots}->{$snapname};
3068         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3069         foreach_drive($snap, sub {
3070             my ($ds, $drive) = @_;
3071             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3072         });
3073     }
3074
3075     foreach my $volid (keys %$volhash) {
3076         &$func($volid, $volhash->{$volid}, @param);
3077     }
3078 }
3079
3080 sub conf_has_serial {
3081     my ($conf) = @_;
3082
3083     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3084         if ($conf->{"serial$i"}) {
3085             return 1;
3086         }
3087     }
3088
3089     return 0;
3090 }
3091
3092 sub vga_conf_has_spice {
3093     my ($vga) = @_;
3094
3095     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3096
3097     return $1 || 1;
3098 }
3099
3100 sub config_to_command {
3101     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3102
3103     my $cmd = [];
3104     my $globalFlags = [];
3105     my $machineFlags = [];
3106     my $rtcFlags = [];
3107     my $cpuFlags = [];
3108     my $devices = [];
3109     my $pciaddr = '';
3110     my $bridges = {};
3111     my $kvmver = kvm_user_version();
3112     my $vernum = 0; # unknown
3113     my $ostype = $conf->{ostype};
3114     my $winversion = windows_version($ostype);
3115     my $kvm = $conf->{kvm} // 1;
3116
3117     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3118
3119     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3120         $vernum = $1*1000000+$2*1000;
3121     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3122         $vernum = $1*1000000+$2*1000+$3;
3123     }
3124
3125     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3126
3127     my $have_ovz = -f '/proc/vz/vestat';
3128
3129     my $q35 = machine_type_is_q35($conf);
3130     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3131     my $machine_type = $forcemachine || $conf->{machine};
3132     my $use_old_bios_files = undef;
3133     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3134
3135     my $cpuunits = defined($conf->{cpuunits}) ?
3136             $conf->{cpuunits} : $defaults->{cpuunits};
3137
3138     push @$cmd, '/usr/bin/kvm';
3139
3140     push @$cmd, '-id', $vmid;
3141
3142     my $vmname = $conf->{name} || "vm$vmid";
3143
3144     push @$cmd, '-name', $vmname;
3145
3146     my $use_virtio = 0;
3147
3148     my $qmpsocket = qmp_socket($vmid);
3149     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3150     push @$cmd, '-mon', "chardev=qmp,mode=control";
3151
3152
3153     push @$cmd, '-pidfile' , pidfile_name($vmid);
3154
3155     push @$cmd, '-daemonize';
3156
3157     if ($conf->{smbios1}) {
3158         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3159     }
3160
3161     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3162         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3163
3164         my $path;
3165         my $format;
3166         if (my $efidisk = $conf->{efidisk0}) {
3167             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3168             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3169             $format = $d->{format};
3170             if ($storeid) {
3171                 $path = PVE::Storage::path($storecfg, $d->{file});
3172                 if (!defined($format)) {
3173                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3174                     $format = qemu_img_format($scfg, $volname);
3175                 }
3176             } else {
3177                 $path = $d->{file};
3178                 die "efidisk format must be specified\n"
3179                     if !defined($format);
3180             }
3181         } else {
3182             warn "no efidisk configured! Using temporary efivars disk.\n";
3183             $path = "/tmp/$vmid-ovmf.fd";
3184             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3185             $format = 'raw';
3186         }
3187
3188         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3189         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3190     }
3191
3192
3193     # add usb controllers
3194     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3195     push @$devices, @usbcontrollers if @usbcontrollers;
3196     my $vga = $conf->{vga};
3197
3198     my $qxlnum = vga_conf_has_spice($vga);
3199     $vga = 'qxl' if $qxlnum;
3200
3201     if (!$vga) {
3202         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3203             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3204         } else {
3205             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3206         }
3207     }
3208
3209     # enable absolute mouse coordinates (needed by vnc)
3210     my $tablet;
3211     if (defined($conf->{tablet})) {
3212         $tablet = $conf->{tablet};
3213     } else {
3214         $tablet = $defaults->{tablet};
3215         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3216         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3217     }
3218
3219     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3220
3221     my $kvm_off = 0;
3222     my $gpu_passthrough;
3223
3224     # host pci devices
3225     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3226         my $d = parse_hostpci($conf->{"hostpci$i"});
3227         next if !$d;
3228
3229         my $pcie = $d->{pcie};
3230         if($pcie){
3231             die "q35 machine model is not enabled" if !$q35;
3232             $pciaddr = print_pcie_addr("hostpci$i");
3233         }else{
3234             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3235         }
3236
3237         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3238         my $romfile = $d->{romfile};
3239
3240         my $xvga = '';
3241         if ($d->{'x-vga'}) {
3242             $xvga = ',x-vga=on';
3243             $kvm_off = 1;
3244             $vga = 'none';
3245             $gpu_passthrough = 1;
3246
3247             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3248                 $xvga = "";
3249             }
3250         }
3251         my $pcidevices = $d->{pciid};
3252         my $multifunction = 1 if @$pcidevices > 1;
3253
3254         my $j=0;
3255         foreach my $pcidevice (@$pcidevices) {
3256
3257             my $id = "hostpci$i";
3258             $id .= ".$j" if $multifunction;
3259             my $addr = $pciaddr;
3260             $addr .= ".$j" if $multifunction;
3261             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3262
3263             if($j == 0){
3264                 $devicestr .= "$rombar$xvga";
3265                 $devicestr .= ",multifunction=on" if $multifunction;
3266                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3267             }
3268
3269             push @$devices, '-device', $devicestr;
3270             $j++;
3271         }
3272     }
3273
3274     # usb devices
3275     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3276     push @$devices, @usbdevices if @usbdevices;
3277     # serial devices
3278     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3279         if (my $path = $conf->{"serial$i"}) {
3280             if ($path eq 'socket') {
3281                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3282                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3283                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3284             } else {
3285                 die "no such serial device\n" if ! -c $path;
3286                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3287                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3288             }
3289         }
3290     }
3291
3292     # parallel devices
3293     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3294         if (my $path = $conf->{"parallel$i"}) {
3295             die "no such parallel device\n" if ! -c $path;
3296             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3297             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3298             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3299         }
3300     }
3301
3302
3303     my $sockets = 1;
3304     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3305     $sockets = $conf->{sockets} if  $conf->{sockets};
3306
3307     my $cores = $conf->{cores} || 1;
3308
3309     my $maxcpus = $sockets * $cores;
3310
3311     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3312
3313     my $allowed_vcpus = $cpuinfo->{cpus};
3314
3315     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3316         if ($allowed_vcpus < $maxcpus);
3317
3318     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3319
3320         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3321         for (my $i = 2; $i <= $vcpus; $i++)  {
3322             my $cpustr = print_cpu_device($conf,$i);
3323             push @$cmd, '-device', $cpustr;
3324         }
3325
3326     } else {
3327
3328         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3329     }
3330     push @$cmd, '-nodefaults';
3331
3332     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3333
3334     my $bootindex_hash = {};
3335     my $i = 1;
3336     foreach my $o (split(//, $bootorder)) {
3337         $bootindex_hash->{$o} = $i*100;
3338         $i++;
3339     }
3340
3341     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3342
3343     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3344
3345     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3346
3347     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3348
3349     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3350         my $socket = vnc_socket($vmid);
3351         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3352     } else {
3353         push @$cmd, '-nographic';
3354     }
3355
3356     # time drift fix
3357     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3358
3359     my $useLocaltime = $conf->{localtime};
3360
3361     if ($winversion >= 5) { # windows
3362         $useLocaltime = 1 if !defined($conf->{localtime});
3363
3364         # use time drift fix when acpi is enabled
3365         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3366             $tdf = 1 if !defined($conf->{tdf});
3367         }
3368     }
3369
3370     if ($winversion >= 6) {
3371         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3372         push @$cmd, '-no-hpet';
3373     }
3374
3375     push @$rtcFlags, 'driftfix=slew' if $tdf;
3376
3377     if (!$kvm) {
3378         push @$machineFlags, 'accel=tcg';
3379     }
3380
3381     if ($machine_type) {
3382         push @$machineFlags, "type=${machine_type}";
3383     }
3384
3385     if ($conf->{startdate}) {
3386         push @$rtcFlags, "base=$conf->{startdate}";
3387     } elsif ($useLocaltime) {
3388         push @$rtcFlags, 'base=localtime';
3389     }
3390
3391     my $cpu = $kvm ? "kvm64" : "qemu64";
3392     if (my $cputype = $conf->{cpu}) {
3393         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3394             or die "Cannot parse cpu description: $cputype\n";
3395         $cpu = $cpuconf->{cputype};
3396         $kvm_off = 1 if $cpuconf->{hidden};
3397
3398         if (defined(my $flags = $cpuconf->{flags})) {
3399             push @$cpuFlags, split(";", $flags);
3400         }
3401     }
3402
3403     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3404
3405     push @$cpuFlags , '-x2apic'
3406         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3407
3408     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3409
3410     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3411
3412     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3413
3414         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3415         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3416     }
3417
3418     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3419
3420     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3421
3422     push @$cpuFlags, 'kvm=off' if $kvm_off;
3423
3424     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3425         die "internal error"; # should not happen
3426
3427     push @$cpuFlags, "vendor=${cpu_vendor}"
3428         if $cpu_vendor ne 'default';
3429
3430     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3431
3432     push @$cmd, '-cpu', $cpu;
3433
3434     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3435
3436     push @$cmd, '-S' if $conf->{freeze};
3437
3438     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3439
3440     # enable sound
3441     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3442     #push @$cmd, '-soundhw', 'es1370';
3443     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3444
3445     if($conf->{agent}) {
3446         my $qgasocket = qmp_socket($vmid, 1);
3447         my $pciaddr = print_pci_addr("qga0", $bridges);
3448         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3449         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3450         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3451     }
3452
3453     my $spice_port;
3454
3455     if ($qxlnum) {
3456         if ($qxlnum > 1) {
3457             if ($winversion){
3458                 for(my $i = 1; $i < $qxlnum; $i++){
3459                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3460                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3461                 }
3462             } else {
3463                 # assume other OS works like Linux
3464                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3465                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3466             }
3467         }
3468
3469         my $pciaddr = print_pci_addr("spice", $bridges);
3470
3471         my $nodename = PVE::INotify::nodename();
3472         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3473         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3474         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3475         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3476         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3477
3478         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3479
3480         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3481         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3482         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3483     }
3484
3485     # enable balloon by default, unless explicitly disabled
3486     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3487         $pciaddr = print_pci_addr("balloon0", $bridges);
3488         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3489     }
3490
3491     if ($conf->{watchdog}) {
3492         my $wdopts = parse_watchdog($conf->{watchdog});
3493         $pciaddr = print_pci_addr("watchdog", $bridges);
3494         my $watchdog = $wdopts->{model} || 'i6300esb';
3495         push @$devices, '-device', "$watchdog$pciaddr";
3496         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3497     }
3498
3499     my $vollist = [];
3500     my $scsicontroller = {};
3501     my $ahcicontroller = {};
3502     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3503
3504     # Add iscsi initiator name if available
3505     if (my $initiator = get_initiator_name()) {
3506         push @$devices, '-iscsi', "initiator-name=$initiator";
3507     }
3508
3509     foreach_drive($conf, sub {
3510         my ($ds, $drive) = @_;
3511
3512         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3513             push @$vollist, $drive->{file};
3514         }
3515
3516         # ignore efidisk here, already added in bios/fw handling code above
3517         return if $drive->{interface} eq 'efidisk';
3518
3519         $use_virtio = 1 if $ds =~ m/^virtio/;
3520
3521         if (drive_is_cdrom ($drive)) {
3522             if ($bootindex_hash->{d}) {
3523                 $drive->{bootindex} = $bootindex_hash->{d};
3524                 $bootindex_hash->{d} += 1;
3525             }
3526         } else {
3527             if ($bootindex_hash->{c}) {
3528                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3529                 $bootindex_hash->{c} += 1;
3530             }
3531         }
3532
3533         if($drive->{interface} eq 'virtio'){
3534            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3535         }
3536
3537         if ($drive->{interface} eq 'scsi') {
3538
3539             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3540
3541             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3542             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3543
3544             my $iothread = '';
3545             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3546                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3547                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3548             } elsif ($drive->{iothread}) {
3549                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3550             }
3551
3552             my $queues = '';
3553             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3554                 $queues = ",num_queues=$drive->{queues}";
3555             }
3556
3557             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3558             $scsicontroller->{$controller}=1;
3559         }
3560
3561         if ($drive->{interface} eq 'sata') {
3562            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3563            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3564            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3565            $ahcicontroller->{$controller}=1;
3566         }
3567
3568         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3569         push @$devices, '-drive',$drive_cmd;
3570         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3571     });
3572
3573     for (my $i = 0; $i < $MAX_NETS; $i++) {
3574          next if !$conf->{"net$i"};
3575          my $d = parse_net($conf->{"net$i"});
3576          next if !$d;
3577
3578          $use_virtio = 1 if $d->{model} eq 'virtio';
3579
3580          if ($bootindex_hash->{n}) {
3581             $d->{bootindex} = $bootindex_hash->{n};
3582             $bootindex_hash->{n} += 1;
3583          }
3584
3585          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3586          push @$devices, '-netdev', $netdevfull;
3587
3588          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3589          push @$devices, '-device', $netdevicefull;
3590     }
3591
3592     if (!$q35) {
3593         # add pci bridges
3594         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3595            $bridges->{1} = 1;
3596            $bridges->{2} = 1;
3597         }
3598
3599         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3600
3601         while (my ($k, $v) = each %$bridges) {
3602             $pciaddr = print_pci_addr("pci.$k");
3603             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3604         }
3605     }
3606
3607     # add custom args
3608     if ($conf->{args}) {
3609         my $aa = PVE::Tools::split_args($conf->{args});
3610         push @$cmd, @$aa;
3611     }
3612
3613     push @$cmd, @$devices;
3614     push @$cmd, '-rtc', join(',', @$rtcFlags)
3615         if scalar(@$rtcFlags);
3616     push @$cmd, '-machine', join(',', @$machineFlags)
3617         if scalar(@$machineFlags);
3618     push @$cmd, '-global', join(',', @$globalFlags)
3619         if scalar(@$globalFlags);
3620
3621     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3622 }
3623
3624 sub vnc_socket {
3625     my ($vmid) = @_;
3626     return "${var_run_tmpdir}/$vmid.vnc";
3627 }
3628
3629 sub spice_port {
3630     my ($vmid) = @_;
3631
3632     my $res = vm_mon_cmd($vmid, 'query-spice');
3633
3634     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3635 }
3636
3637 sub qmp_socket {
3638     my ($vmid, $qga) = @_;
3639     my $sockettype = $qga ? 'qga' : 'qmp';
3640     return "${var_run_tmpdir}/$vmid.$sockettype";
3641 }
3642
3643 sub pidfile_name {
3644     my ($vmid) = @_;
3645     return "${var_run_tmpdir}/$vmid.pid";
3646 }
3647
3648 sub vm_devices_list {
3649     my ($vmid) = @_;
3650
3651     my $res = vm_mon_cmd($vmid, 'query-pci');
3652     my $devices_to_check = [];
3653     my $devices = {};
3654     foreach my $pcibus (@$res) {
3655         push @$devices_to_check, @{$pcibus->{devices}},
3656     }
3657
3658     while (@$devices_to_check) {
3659         my $to_check = [];
3660         for my $d (@$devices_to_check) {
3661             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3662             next if !$d->{'pci_bridge'};
3663
3664             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3665             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3666         }
3667         $devices_to_check = $to_check;
3668     }
3669
3670     my $resblock = vm_mon_cmd($vmid, 'query-block');
3671     foreach my $block (@$resblock) {
3672         if($block->{device} =~ m/^drive-(\S+)/){
3673                 $devices->{$1} = 1;
3674         }
3675     }
3676
3677     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3678     foreach my $mice (@$resmice) {
3679         if ($mice->{name} eq 'QEMU HID Tablet') {
3680             $devices->{tablet} = 1;
3681             last;
3682         }
3683     }
3684
3685     # for usb devices there is no query-usb
3686     # but we can iterate over the entries in
3687     # qom-list path=/machine/peripheral
3688     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3689     foreach my $per (@$resperipheral) {
3690         if ($per->{name} =~ m/^usb\d+$/) {
3691             $devices->{$per->{name}} = 1;
3692         }
3693     }
3694
3695     return $devices;
3696 }
3697
3698 sub vm_deviceplug {
3699     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3700
3701     my $q35 = machine_type_is_q35($conf);
3702
3703     my $devices_list = vm_devices_list($vmid);
3704     return 1 if defined($devices_list->{$deviceid});
3705
3706     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3707
3708     if ($deviceid eq 'tablet') {
3709
3710         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3711
3712     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3713
3714         die "usb hotplug currently not reliable\n";
3715         # since we can't reliably hot unplug all added usb devices
3716         # and usb passthrough disables live migration
3717         # we disable usb hotplugging for now
3718         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3719
3720     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3721
3722         qemu_iothread_add($vmid, $deviceid, $device);
3723
3724         qemu_driveadd($storecfg, $vmid, $device);
3725         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3726
3727         qemu_deviceadd($vmid, $devicefull);
3728         eval { qemu_deviceaddverify($vmid, $deviceid); };
3729         if (my $err = $@) {
3730             eval { qemu_drivedel($vmid, $deviceid); };
3731             warn $@ if $@;
3732             die $err;
3733         }
3734
3735     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3736
3737
3738         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3739         my $pciaddr = print_pci_addr($deviceid);
3740         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3741
3742         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3743
3744         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3745             qemu_iothread_add($vmid, $deviceid, $device);
3746             $devicefull .= ",iothread=iothread-$deviceid";
3747         }
3748
3749         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3750             $devicefull .= ",num_queues=$device->{queues}";
3751         }
3752
3753         qemu_deviceadd($vmid, $devicefull);
3754         qemu_deviceaddverify($vmid, $deviceid);
3755
3756     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3757
3758         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3759         qemu_driveadd($storecfg, $vmid, $device);
3760
3761         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3762         eval { qemu_deviceadd($vmid, $devicefull); };
3763         if (my $err = $@) {
3764             eval { qemu_drivedel($vmid, $deviceid); };
3765             warn $@ if $@;
3766             die $err;
3767         }
3768
3769     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3770
3771         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3772
3773         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3774         my $use_old_bios_files = undef;
3775         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3776
3777         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3778         qemu_deviceadd($vmid, $netdevicefull);
3779         eval { qemu_deviceaddverify($vmid, $deviceid); };
3780         if (my $err = $@) {
3781             eval { qemu_netdevdel($vmid, $deviceid); };
3782             warn $@ if $@;
3783             die $err;
3784         }
3785
3786     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3787
3788         my $bridgeid = $2;
3789         my $pciaddr = print_pci_addr($deviceid);
3790         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3791
3792         qemu_deviceadd($vmid, $devicefull);
3793         qemu_deviceaddverify($vmid, $deviceid);
3794
3795     } else {
3796         die "can't hotplug device '$deviceid'\n";
3797     }
3798
3799     return 1;
3800 }
3801
3802 # fixme: this should raise exceptions on error!
3803 sub vm_deviceunplug {
3804     my ($vmid, $conf, $deviceid) = @_;
3805
3806     my $devices_list = vm_devices_list($vmid);
3807     return 1 if !defined($devices_list->{$deviceid});
3808
3809     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3810
3811     if ($deviceid eq 'tablet') {
3812
3813         qemu_devicedel($vmid, $deviceid);
3814
3815     } elsif ($deviceid =~ m/^usb\d+$/) {
3816
3817         die "usb hotplug currently not reliable\n";
3818         # when unplugging usb devices this way,
3819         # there may be remaining usb controllers/hubs
3820         # so we disable it for now
3821         qemu_devicedel($vmid, $deviceid);
3822         qemu_devicedelverify($vmid, $deviceid);
3823
3824     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3825
3826         qemu_devicedel($vmid, $deviceid);
3827         qemu_devicedelverify($vmid, $deviceid);
3828         qemu_drivedel($vmid, $deviceid);
3829         qemu_iothread_del($conf, $vmid, $deviceid);
3830
3831     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3832
3833         qemu_devicedel($vmid, $deviceid);
3834         qemu_devicedelverify($vmid, $deviceid);
3835         qemu_iothread_del($conf, $vmid, $deviceid);
3836
3837     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3838
3839         qemu_devicedel($vmid, $deviceid);
3840         qemu_drivedel($vmid, $deviceid);
3841         qemu_deletescsihw($conf, $vmid, $deviceid);
3842
3843     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3844
3845         qemu_devicedel($vmid, $deviceid);
3846         qemu_devicedelverify($vmid, $deviceid);
3847         qemu_netdevdel($vmid, $deviceid);
3848
3849     } else {
3850         die "can't unplug device '$deviceid'\n";
3851     }
3852
3853     return 1;
3854 }
3855
3856 sub qemu_deviceadd {
3857     my ($vmid, $devicefull) = @_;
3858
3859     $devicefull = "driver=".$devicefull;
3860     my %options =  split(/[=,]/, $devicefull);
3861
3862     vm_mon_cmd($vmid, "device_add" , %options);
3863 }
3864
3865 sub qemu_devicedel {
3866     my ($vmid, $deviceid) = @_;
3867
3868     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3869 }
3870
3871 sub qemu_iothread_add {
3872     my($vmid, $deviceid, $device) = @_;
3873
3874     if ($device->{iothread}) {
3875         my $iothreads = vm_iothreads_list($vmid);
3876         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3877     }
3878 }
3879
3880 sub qemu_iothread_del {
3881     my($conf, $vmid, $deviceid) = @_;
3882
3883     my $device = parse_drive($deviceid, $conf->{$deviceid});
3884     if ($device->{iothread}) {
3885         my $iothreads = vm_iothreads_list($vmid);
3886         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3887     }
3888 }
3889
3890 sub qemu_objectadd {
3891     my($vmid, $objectid, $qomtype) = @_;
3892
3893     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3894
3895     return 1;
3896 }
3897
3898 sub qemu_objectdel {
3899     my($vmid, $objectid) = @_;
3900
3901     vm_mon_cmd($vmid, "object-del", id => $objectid);
3902
3903     return 1;
3904 }
3905
3906 sub qemu_driveadd {
3907     my ($storecfg, $vmid, $device) = @_;
3908
3909     my $drive = print_drive_full($storecfg, $vmid, $device);
3910     $drive =~ s/\\/\\\\/g;
3911     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3912
3913     # If the command succeeds qemu prints: "OK"
3914     return 1 if $ret =~ m/OK/s;
3915
3916     die "adding drive failed: $ret\n";
3917 }
3918
3919 sub qemu_drivedel {
3920     my($vmid, $deviceid) = @_;
3921
3922     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3923     $ret =~ s/^\s+//;
3924
3925     return 1 if $ret eq "";
3926
3927     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3928     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3929
3930     die "deleting drive $deviceid failed : $ret\n";
3931 }
3932
3933 sub qemu_deviceaddverify {
3934     my ($vmid, $deviceid) = @_;
3935
3936     for (my $i = 0; $i <= 5; $i++) {
3937          my $devices_list = vm_devices_list($vmid);
3938          return 1 if defined($devices_list->{$deviceid});
3939          sleep 1;
3940     }
3941
3942     die "error on hotplug device '$deviceid'\n";
3943 }
3944
3945
3946 sub qemu_devicedelverify {
3947     my ($vmid, $deviceid) = @_;
3948
3949     # need to verify that the device is correctly removed as device_del
3950     # is async and empty return is not reliable
3951
3952     for (my $i = 0; $i <= 5; $i++) {
3953          my $devices_list = vm_devices_list($vmid);
3954          return 1 if !defined($devices_list->{$deviceid});
3955          sleep 1;
3956     }
3957
3958     die "error on hot-unplugging device '$deviceid'\n";
3959 }
3960
3961 sub qemu_findorcreatescsihw {
3962     my ($storecfg, $conf, $vmid, $device) = @_;
3963
3964     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3965
3966     my $scsihwid="$controller_prefix$controller";
3967     my $devices_list = vm_devices_list($vmid);
3968
3969     if(!defined($devices_list->{$scsihwid})) {
3970         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3971     }
3972
3973     return 1;
3974 }
3975
3976 sub qemu_deletescsihw {
3977     my ($conf, $vmid, $opt) = @_;
3978
3979     my $device = parse_drive($opt, $conf->{$opt});
3980
3981     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3982         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3983         return 1;
3984     }
3985
3986     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3987
3988     my $devices_list = vm_devices_list($vmid);
3989     foreach my $opt (keys %{$devices_list}) {
3990         if (PVE::QemuServer::is_valid_drivename($opt)) {
3991             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3992             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3993                 return 1;
3994             }
3995         }
3996     }
3997
3998     my $scsihwid="scsihw$controller";
3999
4000     vm_deviceunplug($vmid, $conf, $scsihwid);
4001
4002     return 1;
4003 }
4004
4005 sub qemu_add_pci_bridge {
4006     my ($storecfg, $conf, $vmid, $device) = @_;
4007
4008     my $bridges = {};
4009
4010     my $bridgeid;
4011
4012     print_pci_addr($device, $bridges);
4013
4014     while (my ($k, $v) = each %$bridges) {
4015         $bridgeid = $k;
4016     }
4017     return 1 if !defined($bridgeid) || $bridgeid < 1;
4018
4019     my $bridge = "pci.$bridgeid";
4020     my $devices_list = vm_devices_list($vmid);
4021
4022     if (!defined($devices_list->{$bridge})) {
4023         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
4024     }
4025
4026     return 1;
4027 }
4028
4029 sub qemu_set_link_status {
4030     my ($vmid, $device, $up) = @_;
4031
4032     vm_mon_cmd($vmid, "set_link", name => $device,
4033                up => $up ? JSON::true : JSON::false);
4034 }
4035
4036 sub qemu_netdevadd {
4037     my ($vmid, $conf, $device, $deviceid) = @_;
4038
4039     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
4040     my %options =  split(/[=,]/, $netdev);
4041
4042     vm_mon_cmd($vmid, "netdev_add",  %options);
4043     return 1;
4044 }
4045
4046 sub qemu_netdevdel {
4047     my ($vmid, $deviceid) = @_;
4048
4049     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4050 }
4051
4052 sub qemu_usb_hotplug {
4053     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4054
4055     return if !$device;
4056
4057     # remove the old one first
4058     vm_deviceunplug($vmid, $conf, $deviceid);
4059
4060     # check if xhci controller is necessary and available
4061     if ($device->{usb3}) {
4062
4063         my $devicelist = vm_devices_list($vmid);
4064
4065         if (!$devicelist->{xhci}) {
4066             my $pciaddr = print_pci_addr("xhci");
4067             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4068         }
4069     }
4070     my $d = parse_usb_device($device->{host});
4071     $d->{usb3} = $device->{usb3};
4072
4073     # add the new one
4074     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4075 }
4076
4077 sub qemu_cpu_hotplug {
4078     my ($vmid, $conf, $vcpus) = @_;
4079
4080     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4081
4082     my $sockets = 1;
4083     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4084     $sockets = $conf->{sockets} if  $conf->{sockets};
4085     my $cores = $conf->{cores} || 1;
4086     my $maxcpus = $sockets * $cores;
4087
4088     $vcpus = $maxcpus if !$vcpus;
4089
4090     die "you can't add more vcpus than maxcpus\n"
4091         if $vcpus > $maxcpus;
4092
4093     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4094
4095     if ($vcpus < $currentvcpus) {
4096
4097         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4098
4099             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4100                 qemu_devicedel($vmid, "cpu$i");
4101                 my $retry = 0;
4102                 my $currentrunningvcpus = undef;
4103                 while (1) {
4104                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4105                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4106                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4107                     $retry++;
4108                     sleep 1;
4109                 }
4110                 #update conf after each succesfull cpu unplug
4111                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4112                 PVE::QemuConfig->write_config($vmid, $conf);
4113             }
4114         } else {
4115             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4116         }
4117
4118         return;
4119     }
4120
4121     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4122     die "vcpus in running vm does not match its configuration\n"
4123         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4124
4125     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4126
4127         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4128             my $cpustr = print_cpu_device($conf, $i);
4129             qemu_deviceadd($vmid, $cpustr);
4130
4131             my $retry = 0;
4132             my $currentrunningvcpus = undef;
4133             while (1) {
4134                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4135                 last if scalar(@{$currentrunningvcpus}) == $i;
4136                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4137                 sleep 1;
4138                 $retry++;
4139             }
4140             #update conf after each succesfull cpu hotplug
4141             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4142             PVE::QemuConfig->write_config($vmid, $conf);
4143         }
4144     } else {
4145
4146         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4147             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4148         }
4149     }
4150 }
4151
4152 sub qemu_block_set_io_throttle {
4153     my ($vmid, $deviceid,
4154         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4155         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4156         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4157         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4158
4159     return if !check_running($vmid) ;
4160
4161     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4162         bps => int($bps),
4163         bps_rd => int($bps_rd),
4164         bps_wr => int($bps_wr),
4165         iops => int($iops),
4166         iops_rd => int($iops_rd),
4167         iops_wr => int($iops_wr),
4168         bps_max => int($bps_max),
4169         bps_rd_max => int($bps_rd_max),
4170         bps_wr_max => int($bps_wr_max),
4171         iops_max => int($iops_max),
4172         iops_rd_max => int($iops_rd_max),
4173         iops_wr_max => int($iops_wr_max),
4174         bps_max_length => int($bps_max_length),
4175         bps_rd_max_length => int($bps_rd_max_length),
4176         bps_wr_max_length => int($bps_wr_max_length),
4177         iops_max_length => int($iops_max_length),
4178         iops_rd_max_length => int($iops_rd_max_length),
4179         iops_wr_max_length => int($iops_wr_max_length),
4180     );
4181
4182 }
4183
4184 # old code, only used to shutdown old VM after update
4185 sub __read_avail {
4186     my ($fh, $timeout) = @_;
4187
4188     my $sel = new IO::Select;
4189     $sel->add($fh);
4190
4191     my $res = '';
4192     my $buf;
4193
4194     my @ready;
4195     while (scalar (@ready = $sel->can_read($timeout))) {
4196         my $count;
4197         if ($count = $fh->sysread($buf, 8192)) {
4198             if ($buf =~ /^(.*)\(qemu\) $/s) {
4199                 $res .= $1;
4200                 last;
4201             } else {
4202                 $res .= $buf;
4203             }
4204         } else {
4205             if (!defined($count)) {
4206                 die "$!\n";
4207             }
4208             last;
4209         }
4210     }
4211
4212     die "monitor read timeout\n" if !scalar(@ready);
4213
4214     return $res;
4215 }
4216
4217 sub qemu_block_resize {
4218     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4219
4220     my $running = check_running($vmid);
4221
4222     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4223
4224     return if !$running;
4225
4226     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4227
4228 }
4229
4230 sub qemu_volume_snapshot {
4231     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4232
4233     my $running = check_running($vmid);
4234
4235     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4236         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4237     } else {
4238         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4239     }
4240 }
4241
4242 sub qemu_volume_snapshot_delete {
4243     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4244
4245     my $running = check_running($vmid);
4246
4247     if($running) {
4248
4249         $running = undef;
4250         my $conf = PVE::QemuConfig->load_config($vmid);
4251         foreach_drive($conf, sub {
4252             my ($ds, $drive) = @_;
4253             $running = 1 if $drive->{file} eq $volid;
4254         });
4255     }
4256
4257     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4258         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4259     } else {
4260         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4261     }
4262 }
4263
4264 sub set_migration_caps {
4265     my ($vmid) = @_;
4266
4267     my $cap_ref = [];
4268
4269     my $enabled_cap = {
4270         "auto-converge" => 1,
4271         "xbzrle" => 1,
4272         "x-rdma-pin-all" => 0,
4273         "zero-blocks" => 0,
4274         "compress" => 0
4275     };
4276
4277     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4278
4279     for my $supported_capability (@$supported_capabilities) {
4280         push @$cap_ref, {
4281             capability => $supported_capability->{capability},
4282             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4283         };
4284     }
4285
4286     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4287 }
4288
4289 my $fast_plug_option = {
4290     'lock' => 1,
4291     'name' => 1,
4292     'onboot' => 1,
4293     'shares' => 1,
4294     'startup' => 1,
4295     'description' => 1,
4296     'protection' => 1,
4297     'vmstatestorage' => 1,
4298 };
4299
4300 # hotplug changes in [PENDING]
4301 # $selection hash can be used to only apply specified options, for
4302 # example: { cores => 1 } (only apply changed 'cores')
4303 # $errors ref is used to return error messages
4304 sub vmconfig_hotplug_pending {
4305     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4306
4307     my $defaults = load_defaults();
4308
4309     # commit values which do not have any impact on running VM first
4310     # Note: those option cannot raise errors, we we do not care about
4311     # $selection and always apply them.
4312
4313     my $add_error = sub {
4314         my ($opt, $msg) = @_;
4315         $errors->{$opt} = "hotplug problem - $msg";
4316     };
4317
4318     my $changes = 0;
4319     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4320         if ($fast_plug_option->{$opt}) {
4321             $conf->{$opt} = $conf->{pending}->{$opt};
4322             delete $conf->{pending}->{$opt};
4323             $changes = 1;
4324         }
4325     }
4326
4327     if ($changes) {
4328         PVE::QemuConfig->write_config($vmid, $conf);
4329         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4330     }
4331
4332     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4333
4334     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4335     while (my ($opt, $force) = each %$pending_delete_hash) {
4336         next if $selection && !$selection->{$opt};
4337         eval {
4338             if ($opt eq 'hotplug') {
4339                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4340             } elsif ($opt eq 'tablet') {
4341                 die "skip\n" if !$hotplug_features->{usb};
4342                 if ($defaults->{tablet}) {
4343                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4344                 } else {
4345                     vm_deviceunplug($vmid, $conf, $opt);
4346                 }
4347             } elsif ($opt =~ m/^usb\d+/) {
4348                 die "skip\n";
4349                 # since we cannot reliably hot unplug usb devices
4350                 # we are disabling it
4351                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4352                 vm_deviceunplug($vmid, $conf, $opt);
4353             } elsif ($opt eq 'vcpus') {
4354                 die "skip\n" if !$hotplug_features->{cpu};
4355                 qemu_cpu_hotplug($vmid, $conf, undef);
4356             } elsif ($opt eq 'balloon') {
4357                 # enable balloon device is not hotpluggable
4358                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4359                 # here we reset the ballooning value to memory
4360                 my $balloon = $conf->{memory} || $defaults->{memory};
4361                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4362             } elsif ($fast_plug_option->{$opt}) {
4363                 # do nothing
4364             } elsif ($opt =~ m/^net(\d+)$/) {
4365                 die "skip\n" if !$hotplug_features->{network};
4366                 vm_deviceunplug($vmid, $conf, $opt);
4367             } elsif (is_valid_drivename($opt)) {
4368                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4369                 vm_deviceunplug($vmid, $conf, $opt);
4370                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4371             } elsif ($opt =~ m/^memory$/) {
4372                 die "skip\n" if !$hotplug_features->{memory};
4373                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4374             } elsif ($opt eq 'cpuunits') {
4375                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4376             } elsif ($opt eq 'cpulimit') {
4377                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4378             } else {
4379                 die "skip\n";
4380             }
4381         };
4382         if (my $err = $@) {
4383             &$add_error($opt, $err) if $err ne "skip\n";
4384         } else {
4385             # save new config if hotplug was successful
4386             delete $conf->{$opt};
4387             vmconfig_undelete_pending_option($conf, $opt);
4388             PVE::QemuConfig->write_config($vmid, $conf);
4389             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4390         }
4391     }
4392
4393     my $apply_pending_cloudinit;
4394     $apply_pending_cloudinit = sub {
4395         my ($key, $value) = @_;
4396         $apply_pending_cloudinit = sub {}; # once is enough
4397
4398         my @cloudinit_opts = keys %$confdesc_cloudinit;
4399         foreach my $opt (keys %{$conf->{pending}}) {
4400             next if !grep { $_ eq $opt } @cloudinit_opts;
4401             $conf->{$opt} = delete $conf->{pending}->{$opt};
4402         }
4403
4404         my $new_conf = { %$conf };
4405         $new_conf->{$key} = $value;
4406         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4407     };
4408
4409     foreach my $opt (keys %{$conf->{pending}}) {
4410         next if $selection && !$selection->{$opt};
4411         my $value = $conf->{pending}->{$opt};
4412         eval {
4413             if ($opt eq 'hotplug') {
4414                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4415             } elsif ($opt eq 'tablet') {
4416                 die "skip\n" if !$hotplug_features->{usb};
4417                 if ($value == 1) {
4418                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4419                 } elsif ($value == 0) {
4420                     vm_deviceunplug($vmid, $conf, $opt);
4421                 }
4422             } elsif ($opt =~ m/^usb\d+$/) {
4423                 die "skip\n";
4424                 # since we cannot reliably hot unplug usb devices
4425                 # we are disabling it
4426                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4427                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4428                 die "skip\n" if !$d;
4429                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4430             } elsif ($opt eq 'vcpus') {
4431                 die "skip\n" if !$hotplug_features->{cpu};
4432                 qemu_cpu_hotplug($vmid, $conf, $value);
4433             } elsif ($opt eq 'balloon') {
4434                 # enable/disable balloning device is not hotpluggable
4435                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4436                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4437                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4438
4439                 # allow manual ballooning if shares is set to zero
4440                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4441                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4442                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4443                 }
4444             } elsif ($opt =~ m/^net(\d+)$/) {
4445                 # some changes can be done without hotplug
4446                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4447                                     $vmid, $opt, $value);
4448             } elsif (is_valid_drivename($opt)) {
4449                 # some changes can be done without hotplug
4450                 my $drive = parse_drive($opt, $value);
4451                 if (drive_is_cloudinit($drive)) {
4452                     &$apply_pending_cloudinit($opt, $value);
4453                 }
4454                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4455                                      $vmid, $opt, $value, 1);
4456             } elsif ($opt =~ m/^memory$/) { #dimms
4457                 die "skip\n" if !$hotplug_features->{memory};
4458                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4459             } elsif ($opt eq 'cpuunits') {
4460                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4461             } elsif ($opt eq 'cpulimit') {
4462                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4463                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4464             } else {
4465                 die "skip\n";  # skip non-hot-pluggable options
4466             }
4467         };
4468         if (my $err = $@) {
4469             &$add_error($opt, $err) if $err ne "skip\n";
4470         } else {
4471             # save new config if hotplug was successful
4472             $conf->{$opt} = $value;
4473             delete $conf->{pending}->{$opt};
4474             PVE::QemuConfig->write_config($vmid, $conf);
4475             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4476         }
4477     }
4478 }
4479
4480 sub try_deallocate_drive {
4481     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4482
4483     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4484         my $volid = $drive->{file};
4485         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4486             my $sid = PVE::Storage::parse_volume_id($volid);
4487             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4488
4489             # check if the disk is really unused
4490             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4491                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4492             PVE::Storage::vdisk_free($storecfg, $volid);
4493             return 1;
4494         } else {
4495             # If vm is not owner of this disk remove from config
4496             return 1;
4497         }
4498     }
4499
4500     return undef;
4501 }
4502
4503 sub vmconfig_delete_or_detach_drive {
4504     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4505
4506     my $drive = parse_drive($opt, $conf->{$opt});
4507
4508     my $rpcenv = PVE::RPCEnvironment::get();
4509     my $authuser = $rpcenv->get_user();
4510
4511     if ($force) {
4512         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4513         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4514     } else {
4515         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4516     }
4517 }
4518
4519 sub vmconfig_apply_pending {
4520     my ($vmid, $conf, $storecfg) = @_;
4521
4522     # cold plug
4523
4524     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4525     while (my ($opt, $force) = each %$pending_delete_hash) {
4526         die "internal error" if $opt =~ m/^unused/;
4527         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4528         if (!defined($conf->{$opt})) {
4529             vmconfig_undelete_pending_option($conf, $opt);
4530             PVE::QemuConfig->write_config($vmid, $conf);
4531         } elsif (is_valid_drivename($opt)) {
4532             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4533             vmconfig_undelete_pending_option($conf, $opt);
4534             delete $conf->{$opt};
4535             PVE::QemuConfig->write_config($vmid, $conf);
4536         } else {
4537             vmconfig_undelete_pending_option($conf, $opt);
4538             delete $conf->{$opt};
4539             PVE::QemuConfig->write_config($vmid, $conf);
4540         }
4541     }
4542
4543     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4544
4545     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4546         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4547
4548         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4549             # skip if nothing changed
4550         } elsif (is_valid_drivename($opt)) {
4551             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4552                 if defined($conf->{$opt});
4553             $conf->{$opt} = $conf->{pending}->{$opt};
4554         } else {
4555             $conf->{$opt} = $conf->{pending}->{$opt};
4556         }
4557
4558         delete $conf->{pending}->{$opt};
4559         PVE::QemuConfig->write_config($vmid, $conf);
4560     }
4561 }
4562
4563 my $safe_num_ne = sub {
4564     my ($a, $b) = @_;
4565
4566     return 0 if !defined($a) && !defined($b);
4567     return 1 if !defined($a);
4568     return 1 if !defined($b);
4569
4570     return $a != $b;
4571 };
4572
4573 my $safe_string_ne = sub {
4574     my ($a, $b) = @_;
4575
4576     return 0 if !defined($a) && !defined($b);
4577     return 1 if !defined($a);
4578     return 1 if !defined($b);
4579
4580     return $a ne $b;
4581 };
4582
4583 sub vmconfig_update_net {
4584     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4585
4586     my $newnet = parse_net($value);
4587
4588     if ($conf->{$opt}) {
4589         my $oldnet = parse_net($conf->{$opt});
4590
4591         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4592             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4593             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4594             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4595
4596             # for non online change, we try to hot-unplug
4597             die "skip\n" if !$hotplug;
4598             vm_deviceunplug($vmid, $conf, $opt);
4599         } else {
4600
4601             die "internal error" if $opt !~ m/net(\d+)/;
4602             my $iface = "tap${vmid}i$1";
4603
4604             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4605                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4606                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4607                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4608                 PVE::Network::tap_unplug($iface);
4609                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4610             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4611                 # Rate can be applied on its own but any change above needs to
4612                 # include the rate in tap_plug since OVS resets everything.
4613                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4614             }
4615
4616             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4617                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4618             }
4619
4620             return 1;
4621         }
4622     }
4623
4624     if ($hotplug) {
4625         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4626     } else {
4627         die "skip\n";
4628     }
4629 }
4630
4631 sub vmconfig_update_disk {
4632     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4633
4634     # fixme: do we need force?
4635
4636     my $drive = parse_drive($opt, $value);
4637
4638     if ($conf->{$opt}) {
4639
4640         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4641
4642             my $media = $drive->{media} || 'disk';
4643             my $oldmedia = $old_drive->{media} || 'disk';
4644             die "unable to change media type\n" if $media ne $oldmedia;
4645
4646             if (!drive_is_cdrom($old_drive)) {
4647
4648                 if ($drive->{file} ne $old_drive->{file}) {
4649
4650                     die "skip\n" if !$hotplug;
4651
4652                     # unplug and register as unused
4653                     vm_deviceunplug($vmid, $conf, $opt);
4654                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4655
4656                 } else {
4657                     # update existing disk
4658
4659                     # skip non hotpluggable value
4660                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4661                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4662                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4663                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4664                         die "skip\n";
4665                     }
4666
4667                     # apply throttle
4668                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4669                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4670                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4671                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4672                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4673                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4674                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4675                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4676                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4677                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4678                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4679                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4680                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4681                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4682                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4683                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4684                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4685                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4686
4687                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4688                                                    ($drive->{mbps} || 0)*1024*1024,
4689                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4690                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4691                                                    $drive->{iops} || 0,
4692                                                    $drive->{iops_rd} || 0,
4693                                                    $drive->{iops_wr} || 0,
4694                                                    ($drive->{mbps_max} || 0)*1024*1024,
4695                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4696                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4697                                                    $drive->{iops_max} || 0,
4698                                                    $drive->{iops_rd_max} || 0,
4699                                                    $drive->{iops_wr_max} || 0,
4700                                                    $drive->{bps_max_length} || 1,
4701                                                    $drive->{bps_rd_max_length} || 1,
4702                                                    $drive->{bps_wr_max_length} || 1,
4703                                                    $drive->{iops_max_length} || 1,
4704                                                    $drive->{iops_rd_max_length} || 1,
4705                                                    $drive->{iops_wr_max_length} || 1);
4706
4707                     }
4708
4709                     return 1;
4710                 }
4711
4712             } else { # cdrom
4713
4714                 if ($drive->{file} eq 'none') {
4715                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4716                     if (drive_is_cloudinit($old_drive)) {
4717                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4718                     }
4719                 } else {
4720                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4721                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4722                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4723                 }
4724
4725                 return 1;
4726             }
4727         }
4728     }
4729
4730     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4731     # hotplug new disks
4732     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4733     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4734 }
4735
4736 sub vm_start {
4737     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4738         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4739
4740     PVE::QemuConfig->lock_config($vmid, sub {
4741         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4742
4743         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4744
4745         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4746
4747         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4748
4749         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4750             vmconfig_apply_pending($vmid, $conf, $storecfg);
4751             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4752         }
4753
4754         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4755
4756         my $defaults = load_defaults();
4757
4758         # set environment variable useful inside network script
4759         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4760
4761         my $local_volumes = {};
4762
4763         if ($targetstorage) {
4764             foreach_drive($conf, sub {
4765                 my ($ds, $drive) = @_;
4766
4767                 return if drive_is_cdrom($drive);
4768
4769                 my $volid = $drive->{file};
4770
4771                 return if !$volid;
4772
4773                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4774
4775                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4776                 return if $scfg->{shared};
4777                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4778             });
4779
4780             my $format = undef;
4781
4782             foreach my $opt (sort keys %$local_volumes) {
4783
4784                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4785                 my $drive = parse_drive($opt, $conf->{$opt});
4786
4787                 #if remote storage is specified, use default format
4788                 if ($targetstorage && $targetstorage ne "1") {
4789                     $storeid = $targetstorage;
4790                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4791                     $format = $defFormat;
4792                 } else {
4793                     #else we use same format than original
4794                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4795                     $format = qemu_img_format($scfg, $volid);
4796                 }
4797
4798                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4799                 my $newdrive = $drive;
4800                 $newdrive->{format} = $format;
4801                 $newdrive->{file} = $newvolid;
4802                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4803                 $local_volumes->{$opt} = $drivestr;
4804                 #pass drive to conf for command line
4805                 $conf->{$opt} = $drivestr;
4806             }
4807         }
4808
4809         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4810
4811         my $migrate_port = 0;
4812         my $migrate_uri;
4813         if ($statefile) {
4814             if ($statefile eq 'tcp') {
4815                 my $localip = "localhost";
4816                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4817                 my $nodename = PVE::INotify::nodename();
4818
4819                 if (!defined($migration_type)) {
4820                     if (defined($datacenterconf->{migration}->{type})) {
4821                         $migration_type = $datacenterconf->{migration}->{type};
4822                     } else {
4823                         $migration_type = 'secure';
4824                     }
4825                 }
4826
4827                 if ($migration_type eq 'insecure') {
4828                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4829                     if ($migrate_network_addr) {
4830                         $localip = $migrate_network_addr;
4831                     } else {
4832                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4833                     }
4834
4835                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4836                 }
4837
4838                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4839                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4840                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4841                 push @$cmd, '-incoming', $migrate_uri;
4842                 push @$cmd, '-S';
4843
4844             } elsif ($statefile eq 'unix') {
4845                 # should be default for secure migrations as a ssh TCP forward
4846                 # tunnel is not deterministic reliable ready and fails regurarly
4847                 # to set up in time, so use UNIX socket forwards
4848                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4849                 unlink $socket_addr;
4850
4851                 $migrate_uri = "unix:$socket_addr";
4852
4853                 push @$cmd, '-incoming', $migrate_uri;
4854                 push @$cmd, '-S';
4855
4856             } else {
4857                 push @$cmd, '-loadstate', $statefile;
4858             }
4859         } elsif ($paused) {
4860             push @$cmd, '-S';
4861         }
4862
4863         # host pci devices
4864         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4865           my $d = parse_hostpci($conf->{"hostpci$i"});
4866           next if !$d;
4867           my $pcidevices = $d->{pciid};
4868           foreach my $pcidevice (@$pcidevices) {
4869                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4870
4871                 my $info = pci_device_info("0000:$pciid");
4872                 die "IOMMU not present\n" if !check_iommu_support();
4873                 die "no pci device info for device '$pciid'\n" if !$info;
4874                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4875                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4876           }
4877         }
4878
4879         PVE::Storage::activate_volumes($storecfg, $vollist);
4880
4881         if (!check_running($vmid, 1)) {
4882             eval {
4883                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4884                     outfunc => sub {}, errfunc => sub {});
4885             };
4886         }
4887
4888         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4889                                                   : $defaults->{cpuunits};
4890
4891         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4892         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4893
4894         my %properties = (
4895             Slice => 'qemu.slice',
4896             KillMode => 'none',
4897             CPUShares => $cpuunits
4898         );
4899
4900         if (my $cpulimit = $conf->{cpulimit}) {
4901             $properties{CPUQuota} = int($cpulimit * 100);
4902         }
4903         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4904
4905         my $run_qemu = sub {
4906             PVE::Tools::run_fork sub {
4907                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4908                 run_command($cmd, %run_params);
4909             };
4910         };
4911
4912         if ($conf->{hugepages}) {
4913
4914             my $code = sub {
4915                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4916                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4917
4918                 PVE::QemuServer::Memory::hugepages_mount();
4919                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4920
4921                 eval { $run_qemu->() };
4922                 if (my $err = $@) {
4923                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4924                     die $err;
4925                 }
4926
4927                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4928             };
4929             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4930
4931         } else {
4932             eval { $run_qemu->() };
4933         }
4934
4935         if (my $err = $@) {
4936             # deactivate volumes if start fails
4937             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4938             die "start failed: $err";
4939         }
4940
4941         print "migration listens on $migrate_uri\n" if $migrate_uri;
4942
4943         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4944             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4945             warn $@ if $@;
4946         }
4947
4948         #start nbd server for storage migration
4949         if ($targetstorage) {
4950             my $nodename = PVE::INotify::nodename();
4951             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4952             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4953             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4954             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4955
4956             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4957
4958             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4959
4960             foreach my $opt (sort keys %$local_volumes) {
4961                 my $volid = $local_volumes->{$opt};
4962                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4963                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4964                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4965             }
4966         }
4967
4968         if ($migratedfrom) {
4969             eval {
4970                 set_migration_caps($vmid);
4971             };
4972             warn $@ if $@;
4973
4974             if ($spice_port) {
4975                 print "spice listens on port $spice_port\n";
4976                 if ($spice_ticket) {
4977                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4978                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4979                 }
4980             }
4981
4982         } else {
4983             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4984                 if !$statefile && $conf->{balloon};
4985
4986             foreach my $opt (keys %$conf) {
4987                 next if $opt !~  m/^net\d+$/;
4988                 my $nicconf = parse_net($conf->{$opt});
4989                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4990             }
4991         }
4992
4993         vm_mon_cmd_nocheck($vmid, 'qom-set',
4994                     path => "machine/peripheral/balloon0",
4995                     property => "guest-stats-polling-interval",
4996                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4997
4998     });
4999 }
5000
5001 sub vm_mon_cmd {
5002     my ($vmid, $execute, %params) = @_;
5003
5004     my $cmd = { execute => $execute, arguments => \%params };
5005     vm_qmp_command($vmid, $cmd);
5006 }
5007
5008 sub vm_mon_cmd_nocheck {
5009     my ($vmid, $execute, %params) = @_;
5010
5011     my $cmd = { execute => $execute, arguments => \%params };
5012     vm_qmp_command($vmid, $cmd, 1);
5013 }
5014
5015 sub vm_qmp_command {
5016     my ($vmid, $cmd, $nocheck) = @_;
5017
5018     my $res;
5019
5020     my $timeout;
5021     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5022         $timeout = $cmd->{arguments}->{timeout};
5023         delete $cmd->{arguments}->{timeout};
5024     }
5025
5026     eval {
5027         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5028         my $sname = qmp_socket($vmid);
5029         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5030             my $qmpclient = PVE::QMPClient->new();
5031
5032             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5033         } else {
5034             die "unable to open monitor socket\n";
5035         }
5036     };
5037     if (my $err = $@) {
5038         syslog("err", "VM $vmid qmp command failed - $err");
5039         die $err;
5040     }
5041
5042     return $res;
5043 }
5044
5045 sub vm_human_monitor_command {
5046     my ($vmid, $cmdline) = @_;
5047
5048     my $res;
5049
5050     my $cmd = {
5051         execute => 'human-monitor-command',
5052         arguments => { 'command-line' => $cmdline},
5053     };
5054
5055     return vm_qmp_command($vmid, $cmd);
5056 }
5057
5058 sub vm_commandline {
5059     my ($storecfg, $vmid) = @_;
5060
5061     my $conf = PVE::QemuConfig->load_config($vmid);
5062
5063     my $defaults = load_defaults();
5064
5065     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5066
5067     return PVE::Tools::cmd2string($cmd);
5068 }
5069
5070 sub vm_reset {
5071     my ($vmid, $skiplock) = @_;
5072
5073     PVE::QemuConfig->lock_config($vmid, sub {
5074
5075         my $conf = PVE::QemuConfig->load_config($vmid);
5076
5077         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5078
5079         vm_mon_cmd($vmid, "system_reset");
5080     });
5081 }
5082
5083 sub get_vm_volumes {
5084     my ($conf) = @_;
5085
5086     my $vollist = [];
5087     foreach_volid($conf, sub {
5088         my ($volid, $attr) = @_;
5089
5090         return if $volid =~ m|^/|;
5091
5092         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5093         return if !$sid;
5094
5095         push @$vollist, $volid;
5096     });
5097
5098     return $vollist;
5099 }
5100
5101 sub vm_stop_cleanup {
5102     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5103
5104     eval {
5105
5106         if (!$keepActive) {
5107             my $vollist = get_vm_volumes($conf);
5108             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5109         }
5110
5111         foreach my $ext (qw(mon qmp pid vnc qga)) {
5112             unlink "/var/run/qemu-server/${vmid}.$ext";
5113         }
5114
5115         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5116     };
5117     warn $@ if $@; # avoid errors - just warn
5118 }
5119
5120 # Note: use $nockeck to skip tests if VM configuration file exists.
5121 # We need that when migration VMs to other nodes (files already moved)
5122 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5123 sub vm_stop {
5124     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5125
5126     $force = 1 if !defined($force) && !$shutdown;
5127
5128     if ($migratedfrom){
5129         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5130         kill 15, $pid if $pid;
5131         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5132         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5133         return;
5134     }
5135
5136     PVE::QemuConfig->lock_config($vmid, sub {
5137
5138         my $pid = check_running($vmid, $nocheck);
5139         return if !$pid;
5140
5141         my $conf;
5142         if (!$nocheck) {
5143             $conf = PVE::QemuConfig->load_config($vmid);
5144             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5145             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5146                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5147                 $timeout = $opts->{down} if $opts->{down};
5148             }
5149         }
5150
5151         $timeout = 60 if !defined($timeout);
5152
5153         eval {
5154             if ($shutdown) {
5155                 if (defined($conf) && $conf->{agent}) {
5156                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5157                 } else {
5158                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5159                 }
5160             } else {
5161                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5162             }
5163         };
5164         my $err = $@;
5165
5166         if (!$err) {
5167             my $count = 0;
5168             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5169                 $count++;
5170                 sleep 1;
5171             }
5172
5173             if ($count >= $timeout) {
5174                 if ($force) {
5175                     warn "VM still running - terminating now with SIGTERM\n";
5176                     kill 15, $pid;
5177                 } else {
5178                     die "VM quit/powerdown failed - got timeout\n";
5179                 }
5180             } else {
5181                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5182                 return;
5183             }
5184         } else {
5185             if ($force) {
5186                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5187                 kill 15, $pid;
5188             } else {
5189                 die "VM quit/powerdown failed\n";
5190             }
5191         }
5192
5193         # wait again
5194         $timeout = 10;
5195
5196         my $count = 0;
5197         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5198             $count++;
5199             sleep 1;
5200         }
5201
5202         if ($count >= $timeout) {
5203             warn "VM still running - terminating now with SIGKILL\n";
5204             kill 9, $pid;
5205             sleep 1;
5206         }
5207
5208         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5209    });
5210 }
5211
5212 sub vm_suspend {
5213     my ($vmid, $skiplock) = @_;
5214
5215     PVE::QemuConfig->lock_config($vmid, sub {
5216
5217         my $conf = PVE::QemuConfig->load_config($vmid);
5218
5219         PVE::QemuConfig->check_lock($conf)
5220             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5221
5222         vm_mon_cmd($vmid, "stop");
5223     });
5224 }
5225
5226 sub vm_resume {
5227     my ($vmid, $skiplock, $nocheck) = @_;
5228
5229     PVE::QemuConfig->lock_config($vmid, sub {
5230
5231         my $res = vm_mon_cmd($vmid, 'query-status');
5232         my $resume_cmd = 'cont';
5233
5234         if ($res->{status} && $res->{status} eq 'suspended') {
5235             $resume_cmd = 'system_wakeup';
5236         }
5237
5238         if (!$nocheck) {
5239
5240             my $conf = PVE::QemuConfig->load_config($vmid);
5241
5242             PVE::QemuConfig->check_lock($conf)
5243                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5244
5245             vm_mon_cmd($vmid, $resume_cmd);
5246
5247         } else {
5248             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5249         }
5250     });
5251 }
5252
5253 sub vm_sendkey {
5254     my ($vmid, $skiplock, $key) = @_;
5255
5256     PVE::QemuConfig->lock_config($vmid, sub {
5257
5258         my $conf = PVE::QemuConfig->load_config($vmid);
5259
5260         # there is no qmp command, so we use the human monitor command
5261         vm_human_monitor_command($vmid, "sendkey $key");
5262     });
5263 }
5264
5265 sub vm_destroy {
5266     my ($storecfg, $vmid, $skiplock) = @_;
5267
5268     PVE::QemuConfig->lock_config($vmid, sub {
5269
5270         my $conf = PVE::QemuConfig->load_config($vmid);
5271
5272         if (!check_running($vmid)) {
5273             destroy_vm($storecfg, $vmid, undef, $skiplock);
5274         } else {
5275             die "VM $vmid is running - destroy failed\n";
5276         }
5277     });
5278 }
5279
5280 # pci helpers
5281
5282 sub file_write {
5283     my ($filename, $buf) = @_;
5284
5285     my $fh = IO::File->new($filename, "w");
5286     return undef if !$fh;
5287
5288     my $res = print $fh $buf;
5289
5290     $fh->close();
5291
5292     return $res;
5293 }
5294
5295 sub pci_device_info {
5296     my ($name) = @_;
5297
5298     my $res;
5299
5300     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5301     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5302
5303     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5304     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5305
5306     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5307     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5308
5309     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5310     return undef if !defined($product) || $product !~ s/^0x//;
5311
5312     $res = {
5313         name => $name,
5314         vendor => $vendor,
5315         product => $product,
5316         domain => $domain,
5317         bus => $bus,
5318         slot => $slot,
5319         func => $func,
5320         irq => $irq,
5321         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5322     };
5323
5324     return $res;
5325 }
5326
5327 sub pci_dev_reset {
5328     my ($dev) = @_;
5329
5330     my $name = $dev->{name};
5331
5332     my $fn = "$pcisysfs/devices/$name/reset";
5333
5334     return file_write($fn, "1");
5335 }
5336
5337 sub pci_dev_bind_to_vfio {
5338     my ($dev) = @_;
5339
5340     my $name = $dev->{name};
5341
5342     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5343
5344     if (!-d $vfio_basedir) {
5345         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5346     }
5347     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5348
5349     my $testdir = "$vfio_basedir/$name";
5350     return 1 if -d $testdir;
5351
5352     my $data = "$dev->{vendor} $dev->{product}";
5353     return undef if !file_write("$vfio_basedir/new_id", $data);
5354
5355     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5356     if (!file_write($fn, $name)) {
5357         return undef if -f $fn;
5358     }
5359
5360     $fn = "$vfio_basedir/bind";
5361     if (! -d $testdir) {
5362         return undef if !file_write($fn, $name);
5363     }
5364
5365     return -d $testdir;
5366 }
5367
5368 sub pci_dev_group_bind_to_vfio {
5369     my ($pciid) = @_;
5370
5371     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5372
5373     if (!-d $vfio_basedir) {
5374         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5375     }
5376     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5377
5378     # get IOMMU group devices
5379     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5380       my @devs = grep /^0000:/, readdir($D);
5381     closedir($D);
5382
5383     foreach my $pciid (@devs) {
5384         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5385
5386         # pci bridges, switches or root ports are not supported
5387         # they have a pci_bus subdirectory so skip them
5388         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5389
5390         my $info = pci_device_info($1);
5391         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5392     }
5393
5394     return 1;
5395 }
5396
5397 # vzdump restore implementaion
5398
5399 sub tar_archive_read_firstfile {
5400     my $archive = shift;
5401
5402     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5403
5404     # try to detect archive type first
5405     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5406         die "unable to open file '$archive'\n";
5407     my $firstfile = <$fh>;
5408     kill 15, $pid;
5409     close $fh;
5410
5411     die "ERROR: archive contaions no data\n" if !$firstfile;
5412     chomp $firstfile;
5413
5414     return $firstfile;
5415 }
5416
5417 sub tar_restore_cleanup {
5418     my ($storecfg, $statfile) = @_;
5419
5420     print STDERR "starting cleanup\n";
5421
5422     if (my $fd = IO::File->new($statfile, "r")) {
5423         while (defined(my $line = <$fd>)) {
5424             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5425                 my $volid = $2;
5426                 eval {
5427                     if ($volid =~ m|^/|) {
5428                         unlink $volid || die 'unlink failed\n';
5429                     } else {
5430                         PVE::Storage::vdisk_free($storecfg, $volid);
5431                     }
5432                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5433                 };
5434                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5435             } else {
5436                 print STDERR "unable to parse line in statfile - $line";
5437             }
5438         }
5439         $fd->close();
5440     }
5441 }
5442
5443 sub restore_archive {
5444     my ($archive, $vmid, $user, $opts) = @_;
5445
5446     my $format = $opts->{format};
5447     my $comp;
5448
5449     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5450         $format = 'tar' if !$format;
5451         $comp = 'gzip';
5452     } elsif ($archive =~ m/\.tar$/) {
5453         $format = 'tar' if !$format;
5454     } elsif ($archive =~ m/.tar.lzo$/) {
5455         $format = 'tar' if !$format;
5456         $comp = 'lzop';
5457     } elsif ($archive =~ m/\.vma$/) {
5458         $format = 'vma' if !$format;
5459     } elsif ($archive =~ m/\.vma\.gz$/) {
5460         $format = 'vma' if !$format;
5461         $comp = 'gzip';
5462     } elsif ($archive =~ m/\.vma\.lzo$/) {
5463         $format = 'vma' if !$format;
5464         $comp = 'lzop';
5465     } else {
5466         $format = 'vma' if !$format; # default
5467     }
5468
5469     # try to detect archive format
5470     if ($format eq 'tar') {
5471         return restore_tar_archive($archive, $vmid, $user, $opts);
5472     } else {
5473         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5474     }
5475 }
5476
5477 sub restore_update_config_line {
5478     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5479
5480     return if $line =~ m/^\#qmdump\#/;
5481     return if $line =~ m/^\#vzdump\#/;
5482     return if $line =~ m/^lock:/;
5483     return if $line =~ m/^unused\d+:/;
5484     return if $line =~ m/^parent:/;
5485     return if $line =~ m/^template:/; # restored VM is never a template
5486
5487     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5488     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5489         # try to convert old 1.X settings
5490         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5491         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5492             my ($model, $macaddr) = split(/\=/, $devconfig);
5493             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5494             my $net = {
5495                 model => $model,
5496                 bridge => "vmbr$ind",
5497                 macaddr => $macaddr,
5498             };
5499             my $netstr = print_net($net);
5500
5501             print $outfd "net$cookie->{netcount}: $netstr\n";
5502             $cookie->{netcount}++;
5503         }
5504     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5505         my ($id, $netstr) = ($1, $2);
5506         my $net = parse_net($netstr);
5507         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5508         $netstr = print_net($net);
5509         print $outfd "$id: $netstr\n";
5510     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5511         my $virtdev = $1;
5512         my $value = $3;
5513         my $di = parse_drive($virtdev, $value);
5514         if (defined($di->{backup}) && !$di->{backup}) {
5515             print $outfd "#$line";
5516         } elsif ($map->{$virtdev}) {
5517             delete $di->{format}; # format can change on restore
5518             $di->{file} = $map->{$virtdev};
5519             $value = print_drive($vmid, $di);
5520             print $outfd "$virtdev: $value\n";
5521         } else {
5522             print $outfd $line;
5523         }
5524     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5525         my ($uuid, $uuid_str);
5526         UUID::generate($uuid);
5527         UUID::unparse($uuid, $uuid_str);
5528         my $smbios1 = parse_smbios1($2);
5529         $smbios1->{uuid} = $uuid_str;
5530         print $outfd $1.print_smbios1($smbios1)."\n";
5531     } else {
5532         print $outfd $line;
5533     }
5534 }
5535
5536 sub scan_volids {
5537     my ($cfg, $vmid) = @_;
5538
5539     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5540
5541     my $volid_hash = {};
5542     foreach my $storeid (keys %$info) {
5543         foreach my $item (@{$info->{$storeid}}) {
5544             next if !($item->{volid} && $item->{size});
5545             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5546             $volid_hash->{$item->{volid}} = $item;
5547         }
5548     }
5549
5550     return $volid_hash;
5551 }
5552
5553 sub is_volume_in_use {
5554     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5555
5556     my $path = PVE::Storage::path($storecfg, $volid);
5557
5558     my $scan_config = sub {
5559         my ($cref, $snapname) = @_;
5560
5561         foreach my $key (keys %$cref) {
5562             my $value = $cref->{$key};
5563             if (is_valid_drivename($key)) {
5564                 next if $skip_drive && $key eq $skip_drive;
5565                 my $drive = parse_drive($key, $value);
5566                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5567                 return 1 if $volid eq $drive->{file};
5568                 if ($drive->{file} =~ m!^/!) {
5569                     return 1 if $drive->{file} eq $path;
5570                 } else {
5571                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5572                     next if !$storeid;
5573                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5574                     next if !$scfg;
5575                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5576                 }
5577             }
5578         }
5579
5580         return 0;
5581     };
5582
5583     return 1 if &$scan_config($conf);
5584
5585     undef $skip_drive;
5586
5587     foreach my $snapname (keys %{$conf->{snapshots}}) {
5588         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5589     }
5590
5591     return 0;
5592 }
5593
5594 sub update_disksize {
5595     my ($vmid, $conf, $volid_hash) = @_;
5596
5597     my $changes;
5598     my $prefix = "VM $vmid:";
5599
5600     # used and unused disks
5601     my $referenced = {};
5602
5603     # Note: it is allowed to define multiple storages with same path (alias), so
5604     # we need to check both 'volid' and real 'path' (two different volid can point
5605     # to the same path).
5606
5607     my $referencedpath = {};
5608
5609     # update size info
5610     foreach my $opt (keys %$conf) {
5611         if (is_valid_drivename($opt)) {
5612             my $drive = parse_drive($opt, $conf->{$opt});
5613             my $volid = $drive->{file};
5614             next if !$volid;
5615
5616             $referenced->{$volid} = 1;
5617             if ($volid_hash->{$volid} &&
5618                 (my $path = $volid_hash->{$volid}->{path})) {
5619                 $referencedpath->{$path} = 1;
5620             }
5621
5622             next if drive_is_cdrom($drive);
5623             next if !$volid_hash->{$volid};
5624
5625             $drive->{size} = $volid_hash->{$volid}->{size};
5626             my $new = print_drive($vmid, $drive);
5627             if ($new ne $conf->{$opt}) {
5628                 $changes = 1;
5629                 $conf->{$opt} = $new;
5630                 print "$prefix update disk '$opt' information.\n";
5631             }
5632         }
5633     }
5634
5635     # remove 'unusedX' entry if volume is used
5636     foreach my $opt (keys %$conf) {
5637         next if $opt !~ m/^unused\d+$/;
5638         my $volid = $conf->{$opt};
5639         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5640         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5641             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5642             $changes = 1;
5643             delete $conf->{$opt};
5644         }
5645
5646         $referenced->{$volid} = 1;
5647         $referencedpath->{$path} = 1 if $path;
5648     }
5649
5650     foreach my $volid (sort keys %$volid_hash) {
5651         next if $volid =~ m/vm-$vmid-state-/;
5652         next if $referenced->{$volid};
5653         my $path = $volid_hash->{$volid}->{path};
5654         next if !$path; # just to be sure
5655         next if $referencedpath->{$path};
5656         $changes = 1;
5657         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5658         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5659         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5660     }
5661
5662     return $changes;
5663 }
5664
5665 sub rescan {
5666     my ($vmid, $nolock, $dryrun) = @_;
5667
5668     my $cfg = PVE::Storage::config();
5669
5670     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5671     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5672     foreach my $stor (keys %{$cfg->{ids}}) {
5673         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5674     }
5675
5676     print "rescan volumes...\n";
5677     my $volid_hash = scan_volids($cfg, $vmid);
5678
5679     my $updatefn =  sub {
5680         my ($vmid) = @_;
5681
5682         my $conf = PVE::QemuConfig->load_config($vmid);
5683
5684         PVE::QemuConfig->check_lock($conf);
5685
5686         my $vm_volids = {};
5687         foreach my $volid (keys %$volid_hash) {
5688             my $info = $volid_hash->{$volid};
5689             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5690         }
5691
5692         my $changes = update_disksize($vmid, $conf, $vm_volids);
5693
5694         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5695     };
5696
5697     if (defined($vmid)) {
5698         if ($nolock) {
5699             &$updatefn($vmid);
5700         } else {
5701             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5702         }
5703     } else {
5704         my $vmlist = config_list();
5705         foreach my $vmid (keys %$vmlist) {
5706             if ($nolock) {
5707                 &$updatefn($vmid);
5708             } else {
5709                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5710             }
5711         }
5712     }
5713 }
5714
5715 sub restore_vma_archive {
5716     my ($archive, $vmid, $user, $opts, $comp) = @_;
5717
5718     my $readfrom = $archive;
5719
5720     my $cfg = PVE::Storage::config();
5721     my $commands = [];
5722     my $bwlimit = $opts->{bwlimit};
5723
5724     my $dbg_cmdstring = '';
5725     my $add_pipe = sub {
5726         my ($cmd) = @_;
5727         push @$commands, $cmd;
5728         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5729         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5730         $readfrom = '-';
5731     };
5732
5733     my $input = undef;
5734     if ($archive eq '-') {
5735         $input = '<&STDIN';
5736     } else {
5737         # If we use a backup from a PVE defined storage we also consider that
5738         # storage's rate limit:
5739         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5740         if (defined($volid)) {
5741             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5742             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5743             if ($readlimit) {
5744                 print STDERR "applying read rate limit: $readlimit\n";
5745                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5746                 $add_pipe->($cstream);
5747             }
5748         }
5749     }
5750
5751     if ($comp) {
5752         my $cmd;
5753         if ($comp eq 'gzip') {
5754             $cmd = ['zcat', $readfrom];
5755         } elsif ($comp eq 'lzop') {
5756             $cmd = ['lzop', '-d', '-c', $readfrom];
5757         } else {
5758             die "unknown compression method '$comp'\n";
5759         }
5760         $add_pipe->($cmd);
5761     }
5762
5763     my $tmpdir = "/var/tmp/vzdumptmp$$";
5764     rmtree $tmpdir;
5765
5766     # disable interrupts (always do cleanups)
5767     local $SIG{INT} =
5768         local $SIG{TERM} =
5769         local $SIG{QUIT} =
5770         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5771
5772     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5773     POSIX::mkfifo($mapfifo, 0600);
5774     my $fifofh;
5775
5776     my $openfifo = sub {
5777         open($fifofh, '>', $mapfifo) || die $!;
5778     };
5779
5780     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5781
5782     my $oldtimeout;
5783     my $timeout = 5;
5784
5785     my $devinfo = {};
5786
5787     my $rpcenv = PVE::RPCEnvironment::get();
5788
5789     my $conffile = PVE::QemuConfig->config_file($vmid);
5790     my $tmpfn = "$conffile.$$.tmp";
5791
5792     # Note: $oldconf is undef if VM does not exists
5793     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5794     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5795
5796     my %storage_limits;
5797
5798     my $print_devmap = sub {
5799         my $virtdev_hash = {};
5800
5801         my $cfgfn = "$tmpdir/qemu-server.conf";
5802
5803         # we can read the config - that is already extracted
5804         my $fh = IO::File->new($cfgfn, "r") ||
5805             "unable to read qemu-server.conf - $!\n";
5806
5807         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5808         if (-f $fwcfgfn) {
5809             my $pve_firewall_dir = '/etc/pve/firewall';
5810             mkdir $pve_firewall_dir; # make sure the dir exists
5811             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5812         }
5813
5814         while (defined(my $line = <$fh>)) {
5815             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5816                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5817                 die "archive does not contain data for drive '$virtdev'\n"
5818                     if !$devinfo->{$devname};
5819                 if (defined($opts->{storage})) {
5820                     $storeid = $opts->{storage} || 'local';
5821                 } elsif (!$storeid) {
5822                     $storeid = 'local';
5823                 }
5824                 $format = 'raw' if !$format;
5825                 $devinfo->{$devname}->{devname} = $devname;
5826                 $devinfo->{$devname}->{virtdev} = $virtdev;
5827                 $devinfo->{$devname}->{format} = $format;
5828                 $devinfo->{$devname}->{storeid} = $storeid;
5829
5830                 # check permission on storage
5831                 my $pool = $opts->{pool}; # todo: do we need that?
5832                 if ($user ne 'root@pam') {
5833                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5834                 }
5835
5836                 $storage_limits{$storeid} = $bwlimit;
5837
5838                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5839             }
5840         }
5841
5842         foreach my $key (keys %storage_limits) {
5843             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5844             next if !$limit;
5845             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5846             $storage_limits{$key} = $limit * 1024;
5847         }
5848
5849         foreach my $devname (keys %$devinfo) {
5850             die "found no device mapping information for device '$devname'\n"
5851                 if !$devinfo->{$devname}->{virtdev};
5852         }
5853
5854         # create empty/temp config
5855         if ($oldconf) {
5856             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5857             foreach_drive($oldconf, sub {
5858                 my ($ds, $drive) = @_;
5859
5860                 return if drive_is_cdrom($drive);
5861
5862                 my $volid = $drive->{file};
5863
5864                 return if !$volid || $volid =~ m|^/|;
5865
5866                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5867                 return if !$path || !$owner || ($owner != $vmid);
5868
5869                 # Note: only delete disk we want to restore
5870                 # other volumes will become unused
5871                 if ($virtdev_hash->{$ds}) {
5872                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5873                     if (my $err = $@) {
5874                         warn $err;
5875                     }
5876                 }
5877             });
5878
5879             # delete vmstate files
5880             # since after the restore we have no snapshots anymore
5881             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5882                 my $snap = $oldconf->{snapshots}->{$snapname};
5883                 if ($snap->{vmstate}) {
5884                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5885                     if (my $err = $@) {
5886                         warn $err;
5887                     }
5888                 }
5889             }
5890         }
5891
5892         my $map = {};
5893         foreach my $virtdev (sort keys %$virtdev_hash) {
5894             my $d = $virtdev_hash->{$virtdev};
5895             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5896             my $storeid = $d->{storeid};
5897             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5898
5899             my $map_opts = '';
5900             if (my $limit = $storage_limits{$storeid}) {
5901                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5902             }
5903
5904             # test if requested format is supported
5905             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5906             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5907             $d->{format} = $defFormat if !$supported;
5908
5909             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5910                                                   $d->{format}, undef, $alloc_size);
5911             print STDERR "new volume ID is '$volid'\n";
5912             $d->{volid} = $volid;
5913             my $path = PVE::Storage::path($cfg, $volid);
5914
5915             PVE::Storage::activate_volumes($cfg,[$volid]);
5916
5917             my $write_zeros = 1;
5918             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5919                 $write_zeros = 0;
5920             }
5921
5922             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5923
5924             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5925             $map->{$virtdev} = $volid;
5926         }
5927
5928         $fh->seek(0, 0) || die "seek failed - $!\n";
5929
5930         my $outfd = new IO::File ($tmpfn, "w") ||
5931             die "unable to write config for VM $vmid\n";
5932
5933         my $cookie = { netcount => 0 };
5934         while (defined(my $line = <$fh>)) {
5935             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5936         }
5937
5938         $fh->close();
5939         $outfd->close();
5940     };
5941
5942     eval {
5943         # enable interrupts
5944         local $SIG{INT} =
5945             local $SIG{TERM} =
5946             local $SIG{QUIT} =
5947             local $SIG{HUP} =
5948             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5949         local $SIG{ALRM} = sub { die "got timeout\n"; };
5950
5951         $oldtimeout = alarm($timeout);
5952
5953         my $parser = sub {
5954             my $line = shift;
5955
5956             print "$line\n";
5957
5958             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5959                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5960                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5961             } elsif ($line =~ m/^CTIME: /) {
5962                 # we correctly received the vma config, so we can disable
5963                 # the timeout now for disk allocation (set to 10 minutes, so
5964                 # that we always timeout if something goes wrong)
5965                 alarm(600);
5966                 &$print_devmap();
5967                 print $fifofh "done\n";
5968                 my $tmp = $oldtimeout || 0;
5969                 $oldtimeout = undef;
5970                 alarm($tmp);
5971                 close($fifofh);
5972             }
5973         };
5974
5975         print "restore vma archive: $dbg_cmdstring\n";
5976         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5977     };
5978     my $err = $@;
5979
5980     alarm($oldtimeout) if $oldtimeout;
5981
5982     my $vollist = [];
5983     foreach my $devname (keys %$devinfo) {
5984         my $volid = $devinfo->{$devname}->{volid};
5985         push @$vollist, $volid if $volid;
5986     }
5987
5988     PVE::Storage::deactivate_volumes($cfg, $vollist);
5989
5990     unlink $mapfifo;
5991
5992     if ($err) {
5993         rmtree $tmpdir;
5994         unlink $tmpfn;
5995
5996         foreach my $devname (keys %$devinfo) {
5997             my $volid = $devinfo->{$devname}->{volid};
5998             next if !$volid;
5999             eval {
6000                 if ($volid =~ m|^/|) {
6001                     unlink $volid || die 'unlink failed\n';
6002                 } else {
6003                     PVE::Storage::vdisk_free($cfg, $volid);
6004                 }
6005                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6006             };
6007             print STDERR "unable to cleanup '$volid' - $@" if $@;
6008         }
6009         die $err;
6010     }
6011
6012     rmtree $tmpdir;
6013
6014     rename($tmpfn, $conffile) ||
6015         die "unable to commit configuration file '$conffile'\n";
6016
6017     PVE::Cluster::cfs_update(); # make sure we read new file
6018
6019     eval { rescan($vmid, 1); };
6020     warn $@ if $@;
6021 }
6022
6023 sub restore_tar_archive {
6024     my ($archive, $vmid, $user, $opts) = @_;
6025
6026     if ($archive ne '-') {
6027         my $firstfile = tar_archive_read_firstfile($archive);
6028         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6029             if $firstfile ne 'qemu-server.conf';
6030     }
6031
6032     my $storecfg = PVE::Storage::config();
6033
6034     # destroy existing data - keep empty config
6035     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6036     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6037
6038     my $tocmd = "/usr/lib/qemu-server/qmextract";
6039
6040     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6041     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6042     $tocmd .= ' --prealloc' if $opts->{prealloc};
6043     $tocmd .= ' --info' if $opts->{info};
6044
6045     # tar option "xf" does not autodetect compression when read from STDIN,
6046     # so we pipe to zcat
6047     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6048         PVE::Tools::shellquote("--to-command=$tocmd");
6049
6050     my $tmpdir = "/var/tmp/vzdumptmp$$";
6051     mkpath $tmpdir;
6052
6053     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6054     local $ENV{VZDUMP_VMID} = $vmid;
6055     local $ENV{VZDUMP_USER} = $user;
6056
6057     my $conffile = PVE::QemuConfig->config_file($vmid);
6058     my $tmpfn = "$conffile.$$.tmp";
6059
6060     # disable interrupts (always do cleanups)
6061     local $SIG{INT} =
6062         local $SIG{TERM} =
6063         local $SIG{QUIT} =
6064         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6065
6066     eval {
6067         # enable interrupts
6068         local $SIG{INT} =
6069             local $SIG{TERM} =
6070             local $SIG{QUIT} =
6071             local $SIG{HUP} =
6072             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6073
6074         if ($archive eq '-') {
6075             print "extracting archive from STDIN\n";
6076             run_command($cmd, input => "<&STDIN");
6077         } else {
6078             print "extracting archive '$archive'\n";
6079             run_command($cmd);
6080         }
6081
6082         return if $opts->{info};
6083
6084         # read new mapping
6085         my $map = {};
6086         my $statfile = "$tmpdir/qmrestore.stat";
6087         if (my $fd = IO::File->new($statfile, "r")) {
6088             while (defined (my $line = <$fd>)) {
6089                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6090                     $map->{$1} = $2 if $1;
6091                 } else {
6092                     print STDERR "unable to parse line in statfile - $line\n";
6093                 }
6094             }
6095             $fd->close();
6096         }
6097
6098         my $confsrc = "$tmpdir/qemu-server.conf";
6099
6100         my $srcfd = new IO::File($confsrc, "r") ||
6101             die "unable to open file '$confsrc'\n";
6102
6103         my $outfd = new IO::File ($tmpfn, "w") ||
6104             die "unable to write config for VM $vmid\n";
6105
6106         my $cookie = { netcount => 0 };
6107         while (defined (my $line = <$srcfd>)) {
6108             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6109         }
6110
6111         $srcfd->close();
6112         $outfd->close();
6113     };
6114     my $err = $@;
6115
6116     if ($err) {
6117
6118         unlink $tmpfn;
6119
6120         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6121
6122         die $err;
6123     }
6124
6125     rmtree $tmpdir;
6126
6127     rename $tmpfn, $conffile ||
6128         die "unable to commit configuration file '$conffile'\n";
6129
6130     PVE::Cluster::cfs_update(); # make sure we read new file
6131
6132     eval { rescan($vmid, 1); };
6133     warn $@ if $@;
6134 };
6135
6136 sub foreach_storage_used_by_vm {
6137     my ($conf, $func) = @_;
6138
6139     my $sidhash = {};
6140
6141     foreach_drive($conf, sub {
6142         my ($ds, $drive) = @_;
6143         return if drive_is_cdrom($drive);
6144
6145         my $volid = $drive->{file};
6146
6147         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6148         $sidhash->{$sid} = $sid if $sid;
6149     });
6150
6151     foreach my $sid (sort keys %$sidhash) {
6152         &$func($sid);
6153     }
6154 }
6155
6156 sub do_snapshots_with_qemu {
6157     my ($storecfg, $volid) = @_;
6158
6159     my $storage_name = PVE::Storage::parse_volume_id($volid);
6160
6161     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6162         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6163         return 1;
6164     }
6165
6166     if ($volid =~ m/\.(qcow2|qed)$/){
6167         return 1;
6168     }
6169
6170     return undef;
6171 }
6172
6173 sub qga_check_running {
6174     my ($vmid, $nowarn) = @_;
6175
6176     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6177     if ($@) {
6178         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6179         return 0;
6180     }
6181     return 1;
6182 }
6183
6184 sub template_create {
6185     my ($vmid, $conf, $disk) = @_;
6186
6187     my $storecfg = PVE::Storage::config();
6188
6189     foreach_drive($conf, sub {
6190         my ($ds, $drive) = @_;
6191
6192         return if drive_is_cdrom($drive);
6193         return if $disk && $ds ne $disk;
6194
6195         my $volid = $drive->{file};
6196         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6197
6198         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6199         $drive->{file} = $voliddst;
6200         $conf->{$ds} = print_drive($vmid, $drive);
6201         PVE::QemuConfig->write_config($vmid, $conf);
6202     });
6203 }
6204
6205 sub qemu_img_convert {
6206     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6207
6208     my $storecfg = PVE::Storage::config();
6209     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6210     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6211
6212     if ($src_storeid && $dst_storeid) {
6213
6214         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6215
6216         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6217         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6218
6219         my $src_format = qemu_img_format($src_scfg, $src_volname);
6220         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6221
6222         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6223         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6224
6225         my $cmd = [];
6226         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6227         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6228         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6229         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6230         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6231         if ($is_zero_initialized) {
6232             push @$cmd, "zeroinit:$dst_path";
6233         } else {
6234             push @$cmd, $dst_path;
6235         }
6236
6237         my $parser = sub {
6238             my $line = shift;
6239             if($line =~ m/\((\S+)\/100\%\)/){
6240                 my $percent = $1;
6241                 my $transferred = int($size * $percent / 100);
6242                 my $remaining = $size - $transferred;
6243
6244                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6245             }
6246
6247         };
6248
6249         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6250         my $err = $@;
6251         die "copy failed: $err" if $err;
6252     }
6253 }
6254
6255 sub qemu_img_format {
6256     my ($scfg, $volname) = @_;
6257
6258     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6259         return $1;
6260     } else {
6261         return "raw";
6262     }
6263 }
6264
6265 sub qemu_drive_mirror {
6266     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6267
6268     $jobs = {} if !$jobs;
6269
6270     my $qemu_target;
6271     my $format;
6272     $jobs->{"drive-$drive"} = {};
6273
6274     if ($dst_volid =~ /^nbd:/) {
6275         $qemu_target = $dst_volid;
6276         $format = "nbd";
6277     } else {
6278         my $storecfg = PVE::Storage::config();
6279         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6280
6281         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6282
6283         $format = qemu_img_format($dst_scfg, $dst_volname);
6284
6285         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6286
6287         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6288     }
6289
6290     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6291     $opts->{format} = $format if $format;
6292
6293     print "drive mirror is starting for drive-$drive\n";
6294
6295     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6296
6297     if (my $err = $@) {
6298         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6299         die "mirroring error: $err";
6300     }
6301
6302     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6303 }
6304
6305 sub qemu_drive_mirror_monitor {
6306     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6307
6308     eval {
6309         my $err_complete = 0;
6310
6311         while (1) {
6312             die "storage migration timed out\n" if $err_complete > 300;
6313
6314             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6315
6316             my $running_mirror_jobs = {};
6317             foreach my $stat (@$stats) {
6318                 next if $stat->{type} ne 'mirror';
6319                 $running_mirror_jobs->{$stat->{device}} = $stat;
6320             }
6321
6322             my $readycounter = 0;
6323
6324             foreach my $job (keys %$jobs) {
6325
6326                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6327                     print "$job : finished\n";
6328                     delete $jobs->{$job};
6329                     next;
6330                 }
6331
6332                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6333
6334                 my $busy = $running_mirror_jobs->{$job}->{busy};
6335                 my $ready = $running_mirror_jobs->{$job}->{ready};
6336                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6337                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6338                     my $remaining = $total - $transferred;
6339                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6340
6341                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6342                 }
6343
6344                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6345             }
6346
6347             last if scalar(keys %$jobs) == 0;
6348
6349             if ($readycounter == scalar(keys %$jobs)) {
6350                 print "all mirroring jobs are ready \n";
6351                 last if $skipcomplete; #do the complete later
6352
6353                 if ($vmiddst && $vmiddst != $vmid) {
6354                     my $agent_running = $qga && qga_check_running($vmid);
6355                     if ($agent_running) {
6356                         print "freeze filesystem\n";
6357                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6358                     } else {
6359                         print "suspend vm\n";
6360                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6361                     }
6362
6363                     # if we clone a disk for a new target vm, we don't switch the disk
6364                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6365
6366                     if ($agent_running) {
6367                         print "unfreeze filesystem\n";
6368                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6369                     } else {
6370                         print "resume vm\n";
6371                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6372                     }
6373
6374                     last;
6375                 } else {
6376
6377                     foreach my $job (keys %$jobs) {
6378                         # try to switch the disk if source and destination are on the same guest
6379                         print "$job: Completing block job...\n";
6380
6381                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6382                         if ($@ =~ m/cannot be completed/) {
6383                             print "$job: Block job cannot be completed, try again.\n";
6384                             $err_complete++;
6385                         }else {
6386                             print "$job: Completed successfully.\n";
6387                             $jobs->{$job}->{complete} = 1;
6388                         }
6389                     }
6390                 }
6391             }
6392             sleep 1;
6393         }
6394     };
6395     my $err = $@;
6396
6397     if ($err) {
6398         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6399         die "mirroring error: $err";
6400     }
6401
6402 }
6403
6404 sub qemu_blockjobs_cancel {
6405     my ($vmid, $jobs) = @_;
6406
6407     foreach my $job (keys %$jobs) {
6408         print "$job: Cancelling block job\n";
6409         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6410         $jobs->{$job}->{cancel} = 1;
6411     }
6412
6413     while (1) {
6414         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6415
6416         my $running_jobs = {};
6417         foreach my $stat (@$stats) {
6418             $running_jobs->{$stat->{device}} = $stat;
6419         }
6420
6421         foreach my $job (keys %$jobs) {
6422
6423             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6424                 print "$job: Done.\n";
6425                 delete $jobs->{$job};
6426             }
6427         }
6428
6429         last if scalar(keys %$jobs) == 0;
6430
6431         sleep 1;
6432     }
6433 }
6434
6435 sub clone_disk {
6436     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6437         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6438
6439     my $newvolid;
6440
6441     if (!$full) {
6442         print "create linked clone of drive $drivename ($drive->{file})\n";
6443         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6444         push @$newvollist, $newvolid;
6445     } else {
6446
6447         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6448         $storeid = $storage if $storage;
6449
6450         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6451         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6452
6453         print "create full clone of drive $drivename ($drive->{file})\n";
6454         my $name = undef;
6455         if (drive_is_cloudinit($drive)) {
6456             $name = "vm-$newvmid-cloudinit";
6457             # cloudinit only supports raw and qcow2 atm:
6458             if ($dst_format eq 'qcow2') {
6459                 $name .= '.qcow2';
6460             } elsif ($dst_format ne 'raw') {
6461                 die "clone: unhandled format for cloudinit image\n";
6462             }
6463         }
6464         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6465         push @$newvollist, $newvolid;
6466
6467         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6468
6469         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6470         if (!$running || $snapname) {
6471             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6472         } else {
6473
6474             my $kvmver = get_running_qemu_version ($vmid);
6475             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6476                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6477                     if $drive->{iothread};
6478             }
6479
6480             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6481         }
6482     }
6483
6484     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6485
6486     my $disk = $drive;
6487     $disk->{format} = undef;
6488     $disk->{file} = $newvolid;
6489     $disk->{size} = $size;
6490
6491     return $disk;
6492 }
6493
6494 # this only works if VM is running
6495 sub get_current_qemu_machine {
6496     my ($vmid) = @_;
6497
6498     my $cmd = { execute => 'query-machines', arguments => {} };
6499     my $res = vm_qmp_command($vmid, $cmd);
6500
6501     my ($current, $default);
6502     foreach my $e (@$res) {
6503         $default = $e->{name} if $e->{'is-default'};
6504         $current = $e->{name} if $e->{'is-current'};
6505     }
6506
6507     # fallback to the default machine if current is not supported by qemu
6508     return $current || $default || 'pc';
6509 }
6510
6511 sub get_running_qemu_version {
6512     my ($vmid) = @_;
6513     my $cmd = { execute => 'query-version', arguments => {} };
6514     my $res = vm_qmp_command($vmid, $cmd);
6515     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6516 }
6517
6518 sub qemu_machine_feature_enabled {
6519     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6520
6521     my $current_major;
6522     my $current_minor;
6523
6524     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6525
6526         $current_major = $3;
6527         $current_minor = $4;
6528
6529     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6530
6531         $current_major = $1;
6532         $current_minor = $2;
6533     }
6534
6535     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6536
6537
6538 }
6539
6540 sub qemu_machine_pxe {
6541     my ($vmid, $conf, $machine) = @_;
6542
6543     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6544
6545     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6546         $machine .= '.pxe';
6547     }
6548
6549     return $machine;
6550 }
6551
6552 sub qemu_use_old_bios_files {
6553     my ($machine_type) = @_;
6554
6555     return if !$machine_type;
6556
6557     my $use_old_bios_files = undef;
6558
6559     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6560         $machine_type = $1;
6561         $use_old_bios_files = 1;
6562     } else {
6563         my $kvmver = kvm_user_version();
6564         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6565         # load new efi bios files on migration. So this hack is required to allow
6566         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6567         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6568         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6569     }
6570
6571     return ($use_old_bios_files, $machine_type);
6572 }
6573
6574 sub create_efidisk {
6575     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6576
6577     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6578
6579     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6580     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6581     PVE::Storage::activate_volumes($storecfg, [$volid]);
6582
6583     my $path = PVE::Storage::path($storecfg, $volid);
6584     eval {
6585         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6586     };
6587     die "Copying EFI vars image failed: $@" if $@;
6588
6589     return ($volid, $vars_size);
6590 }
6591
6592 sub lspci {
6593
6594     my $devices = {};
6595
6596     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6597             my (undef, $id, $function) = @_;
6598             my $res = { id => $id, function => $function};
6599             push @{$devices->{$id}}, $res;
6600     });
6601
6602     # Entries should be sorted by functions.
6603     foreach my $id (keys %$devices) {
6604         my $dev = $devices->{$id};
6605         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6606     }
6607
6608     return $devices;
6609 }
6610
6611 sub vm_iothreads_list {
6612     my ($vmid) = @_;
6613
6614     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6615
6616     my $iothreads = {};
6617     foreach my $iothread (@$res) {
6618         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6619     }
6620
6621     return $iothreads;
6622 }
6623
6624 sub scsihw_infos {
6625     my ($conf, $drive) = @_;
6626
6627     my $maxdev = 0;
6628
6629     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6630         $maxdev = 7;
6631     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6632         $maxdev = 1;
6633     } else {
6634         $maxdev = 256;
6635     }
6636
6637     my $controller = int($drive->{index} / $maxdev);
6638     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6639
6640     return ($maxdev, $controller, $controller_prefix);
6641 }
6642
6643 sub add_hyperv_enlightenments {
6644     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6645
6646     return if $winversion < 6;
6647     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6648
6649     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6650
6651     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6652         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6653         push @$cpuFlags , 'hv_vapic';
6654         push @$cpuFlags , 'hv_time';
6655     } else {
6656         push @$cpuFlags , 'hv_spinlocks=0xffff';
6657     }
6658
6659     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6660         push @$cpuFlags , 'hv_reset';
6661         push @$cpuFlags , 'hv_vpindex';
6662         push @$cpuFlags , 'hv_runtime';
6663     }
6664
6665     if ($winversion >= 7) {
6666         push @$cpuFlags , 'hv_relaxed';
6667     }
6668 }
6669
6670 sub windows_version {
6671     my ($ostype) = @_;
6672
6673     return 0 if !$ostype;
6674
6675     my $winversion = 0;
6676
6677     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6678         $winversion = 5;
6679     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6680         $winversion = 6;
6681     } elsif ($ostype =~ m/^win(\d+)$/) {
6682         $winversion = $1;
6683     }
6684
6685     return $winversion;
6686 }
6687
6688 sub resolve_dst_disk_format {
6689         my ($storecfg, $storeid, $src_volname, $format) = @_;
6690         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6691
6692         if (!$format) {
6693             # if no target format is specified, use the source disk format as hint
6694             if ($src_volname) {
6695                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6696                 $format = qemu_img_format($scfg, $src_volname);
6697             } else {
6698                 return $defFormat;
6699             }
6700         }
6701
6702         # test if requested format is supported - else use default
6703         my $supported = grep { $_ eq $format } @$validFormats;
6704         $format = $defFormat if !$supported;
6705         return $format;
6706 }
6707
6708 sub resolve_first_disk {
6709     my $conf = shift;
6710     my @disks = PVE::QemuServer::valid_drive_names();
6711     my $firstdisk;
6712     foreach my $ds (reverse @disks) {
6713         next if !$conf->{$ds};
6714         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6715         next if PVE::QemuServer::drive_is_cdrom($disk);
6716         $firstdisk = $ds;
6717     }
6718     return $firstdisk;
6719 }
6720
6721 sub generate_smbios1_uuid {
6722     my ($uuid, $uuid_str);
6723     UUID::generate($uuid);
6724     UUID::unparse($uuid, $uuid_str);
6725     return "uuid=$uuid_str";
6726 }
6727
6728 # bash completion helper
6729
6730 sub complete_backup_archives {
6731     my ($cmdname, $pname, $cvalue) = @_;
6732
6733     my $cfg = PVE::Storage::config();
6734
6735     my $storeid;
6736
6737     if ($cvalue =~ m/^([^:]+):/) {
6738         $storeid = $1;
6739     }
6740
6741     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6742
6743     my $res = [];
6744     foreach my $id (keys %$data) {
6745         foreach my $item (@{$data->{$id}}) {
6746             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6747             push @$res, $item->{volid} if defined($item->{volid});
6748         }
6749     }
6750
6751     return $res;
6752 }
6753
6754 my $complete_vmid_full = sub {
6755     my ($running) = @_;
6756
6757     my $idlist = vmstatus();
6758
6759     my $res = [];
6760
6761     foreach my $id (keys %$idlist) {
6762         my $d = $idlist->{$id};
6763         if (defined($running)) {
6764             next if $d->{template};
6765             next if $running && $d->{status} ne 'running';
6766             next if !$running && $d->{status} eq 'running';
6767         }
6768         push @$res, $id;
6769
6770     }
6771     return $res;
6772 };
6773
6774 sub complete_vmid {
6775     return &$complete_vmid_full();
6776 }
6777
6778 sub complete_vmid_stopped {
6779     return &$complete_vmid_full(0);
6780 }
6781
6782 sub complete_vmid_running {
6783     return &$complete_vmid_full(1);
6784 }
6785
6786 sub complete_storage {
6787
6788     my $cfg = PVE::Storage::config();
6789     my $ids = $cfg->{ids};
6790
6791     my $res = [];
6792     foreach my $sid (keys %$ids) {
6793         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6794         next if !$ids->{$sid}->{content}->{images};
6795         push @$res, $sid;
6796     }
6797
6798     return $res;
6799 }
6800
6801 sub nbd_stop {
6802     my ($vmid) = @_;
6803
6804     vm_mon_cmd($vmid, 'nbd-server-stop');
6805 }
6806
6807 1;