PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 #no warnings 'redefine';
  84
  85 sub cgroups_write {
  86    my ($controller, $vmid, $option, $value) = @_;
  87
  88    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  89    PVE::ProcFSTools::write_proc_entry($path, $value);
  90
  91 }
  92
  93 my $nodename = PVE::INotify::nodename();
  94
  95 mkdir "/etc/pve/nodes/$nodename";
  96 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  97 mkdir $confdir;
  98
  99 my $var_run_tmpdir = "/var/run/qemu-server";
 100 mkdir $var_run_tmpdir;
 101
 102 my $lock_dir = "/var/lock/qemu-server";
 103 mkdir $lock_dir;
 104
 105 my $pcisysfs = "/sys/bus/pci";
 106
 107 my $cpu_vendor_list = {
 108     # Intel CPUs
 109     486 => 'GenuineIntel',
 110     pentium => 'GenuineIntel',
 111     pentium2  => 'GenuineIntel',
 112     pentium3  => 'GenuineIntel',
 113     coreduo => 'GenuineIntel',
 114     core2duo => 'GenuineIntel',
 115     Conroe  => 'GenuineIntel',
 116     Penryn  => 'GenuineIntel',
 117     Nehalem  => 'GenuineIntel',
 118     'Nehalem-IBRS'  => 'GenuineIntel',
 119     Westmere => 'GenuineIntel',
 120     'Westmere-IBRS' => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     'SandyBridge-IBRS' => 'GenuineIntel',
 123     IvyBridge => 'GenuineIntel',
 124     'IvyBridge-IBRS' => 'GenuineIntel',
 125     Haswell => 'GenuineIntel',
 126     'Haswell-IBRS' => 'GenuineIntel',
 127     'Haswell-noTSX' => 'GenuineIntel',
 128     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 129     Broadwell => 'GenuineIntel',
 130     'Broadwell-IBRS' => 'GenuineIntel',
 131     'Broadwell-noTSX' => 'GenuineIntel',
 132     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 133     'Skylake-Client' => 'GenuineIntel',
 134     'Skylake-Client-IBRS' => 'GenuineIntel',
 135     'Skylake-Server' => 'GenuineIntel',
 136     'Skylake-Server-IBRS' => 'GenuineIntel',
 137
 138     # AMD CPUs
 139     athlon => 'AuthenticAMD',
 140     phenom  => 'AuthenticAMD',
 141     Opteron_G1  => 'AuthenticAMD',
 142     Opteron_G2  => 'AuthenticAMD',
 143     Opteron_G3  => 'AuthenticAMD',
 144     Opteron_G4  => 'AuthenticAMD',
 145     Opteron_G5  => 'AuthenticAMD',
 146     EPYC => 'AuthenticAMD',
 147     'EPYC-IBPB' => 'AuthenticAMD',
 148
 149     # generic types, use vendor from host node
 150     host => 'default',
 151     kvm32 => 'default',
 152     kvm64 => 'default',
 153     qemu32 => 'default',
 154     qemu64 => 'default',
 155     max => 'default',
 156 };
 157
 158 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 159
 160 my $cpu_fmt = {
 161     cputype => {
 162         description => "Emulated CPU type.",
 163         type => 'string',
 164         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 165         default => 'kvm64',
 166         default_key => 1,
 167     },
 168     hidden => {
 169         description => "Do not identify as a KVM virtual machine.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 0
 173     },
 174     flags => {
 175         description => "List of additional CPU flags separated by ';'."
 176                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 177                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 178         format_description => '+FLAG[;-FLAG...]',
 179         type => 'string',
 180         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 181         optional => 1,
 182     },
 183 };
 184
 185 my $watchdog_fmt = {
 186     model => {
 187         default_key => 1,
 188         type => 'string',
 189         enum => [qw(i6300esb ib700)],
 190         description => "Watchdog type to emulate.",
 191         default => 'i6300esb',
 192         optional => 1,
 193     },
 194     action => {
 195         type => 'string',
 196         enum => [qw(reset shutdown poweroff pause debug none)],
 197         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 198         optional => 1,
 199     },
 200 };
 201 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 202
 203 my $confdesc = {
 204     onboot => {
 205         optional => 1,
 206         type => 'boolean',
 207         description => "Specifies whether a VM will be started during system bootup.",
 208         default => 0,
 209     },
 210     autostart => {
 211         optional => 1,
 212         type => 'boolean',
 213         description => "Automatic restart after crash (currently ignored).",
 214         default => 0,
 215     },
 216     hotplug => {
 217         optional => 1,
 218         type => 'string', format => 'pve-hotplug-features',
 219         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 220         default => 'network,disk,usb',
 221     },
 222     reboot => {
 223         optional => 1,
 224         type => 'boolean',
 225         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 226         default => 1,
 227     },
 228     lock => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Lock/unlock the VM.",
 232         enum => [qw(migrate backup snapshot rollback)],
 233     },
 234     cpulimit => {
 235         optional => 1,
 236         type => 'number',
 237         description => "Limit of CPU usage.",
 238         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 239         minimum => 0,
 240         maximum => 128,
 241         default => 0,
 242     },
 243     cpuunits => {
 244         optional => 1,
 245         type => 'integer',
 246         description => "CPU weight for a VM.",
 247         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 248         minimum => 2,
 249         maximum => 262144,
 250         default => 1024,
 251     },
 252     memory => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 256         minimum => 16,
 257         default => 512,
 258     },
 259     balloon => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 263         minimum => 0,
 264     },
 265     shares => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 269         minimum => 0,
 270         maximum => 50000,
 271         default => 1000,
 272     },
 273     keyboard => {
 274         optional => 1,
 275         type => 'string',
 276         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 277                        "It should not be necessary to set it.",
 278         enum => PVE::Tools::kvmkeymaplist(),
 279         default => undef,
 280     },
 281     name => {
 282         optional => 1,
 283         type => 'string', format => 'dns-name',
 284         description => "Set a name for the VM. Only used on the configuration web interface.",
 285     },
 286     scsihw => {
 287         optional => 1,
 288         type => 'string',
 289         description => "SCSI controller model",
 290         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 291         default => 'lsi',
 292     },
 293     description => {
 294         optional => 1,
 295         type => 'string',
 296         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 297     },
 298     ostype => {
 299         optional => 1,
 300         type => 'string',
 301         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 302         description => "Specify guest operating system.",
 303         verbose_description => <<EODESC,
 304 Specify guest operating system. This is used to enable special
 305 optimization/features for specific operating systems:
 306
 307 [horizontal]
 308 other;; unspecified OS
 309 wxp;; Microsoft Windows XP
 310 w2k;; Microsoft Windows 2000
 311 w2k3;; Microsoft Windows 2003
 312 w2k8;; Microsoft Windows 2008
 313 wvista;; Microsoft Windows Vista
 314 win7;; Microsoft Windows 7
 315 win8;; Microsoft Windows 8/2012/2012r2
 316 win10;; Microsoft Windows 10/2016
 317 l24;; Linux 2.4 Kernel
 318 l26;; Linux 2.6/3.X Kernel
 319 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 320 EODESC
 321     },
 322     boot => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 326         pattern => '[acdn]{1,4}',
 327         default => 'cdn',
 328     },
 329     bootdisk => {
 330         optional => 1,
 331         type => 'string', format => 'pve-qm-bootdisk',
 332         description => "Enable booting from specified disk.",
 333         pattern => '(ide|sata|scsi|virtio)\d+',
 334     },
 335     smp => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPUs. Please use option -sockets instead.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     sockets => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of CPU sockets.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     cores => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "The number of cores per socket.",
 353         minimum => 1,
 354         default => 1,
 355     },
 356     numa => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable NUMA.",
 360         default => 0,
 361     },
 362     hugepages => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Enable/disable hugepages memory.",
 366         enum => [qw(any 2 1024)],
 367     },
 368     vcpus => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Number of hotplugged vcpus.",
 372         minimum => 1,
 373         default => 0,
 374     },
 375     acpi => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable ACPI.",
 379         default => 1,
 380     },
 381     agent => {
 382         optional => 1,
 383         type => 'boolean',
 384         description => "Enable/disable Qemu GuestAgent.",
 385         default => 0,
 386     },
 387     kvm => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Enable/disable KVM hardware virtualization.",
 391         default => 1,
 392     },
 393     tdf => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Enable/disable time drift fix.",
 397         default => 0,
 398     },
 399     localtime => {
 400         optional => 1,
 401         type => 'boolean',
 402         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 403     },
 404     freeze => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 408     },
 409     vga => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Select the VGA type.",
 413         verbose_description => "Select the VGA type. If you want to use high resolution" .
 414             " modes (>= 1280x1024x16) then you should use the options " .
 415             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 416             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 417             "display sever. For win* OS you can select how many independent " .
 418             "displays you want, Linux guests can add displays them self. " .
 419             "You can also run without any graphic card, using a serial device" .
 420             " as terminal.",
 421         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 422     },
 423     watchdog => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-watchdog',
 426         description => "Create a virtual hardware watchdog device.",
 427         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 428             " (by a guest action), the watchdog must be periodically polled " .
 429             "by an agent inside the guest or else the watchdog will reset " .
 430             "the guest (or execute the respective action specified)",
 431     },
 432     startdate => {
 433         optional => 1,
 434         type => 'string',
 435         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 436         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 437         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 438         default => 'now',
 439     },
 440     startup =>  get_standard_option('pve-startup-order'),
 441     template => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable Template.",
 445         default => 0,
 446     },
 447     args => {
 448         optional => 1,
 449         type => 'string',
 450         description => "Arbitrary arguments passed to kvm.",
 451         verbose_description => <<EODESCR,
 452 Arbitrary arguments passed to kvm, for example:
 453
 454 args: -no-reboot -no-hpet
 455
 456 NOTE: this option is for experts only.
 457 EODESCR
 458     },
 459     tablet => {
 460         optional => 1,
 461         type => 'boolean',
 462         default => 1,
 463         description => "Enable/disable the USB tablet device.",
 464         verbose_description => "Enable/disable the USB tablet device. This device is " .
 465             "usually needed to allow absolute mouse positioning with VNC. " .
 466             "Else the mouse runs out of sync with normal VNC clients. " .
 467             "If you're running lots of console-only guests on one host, " .
 468             "you may consider disabling this to save some context switches. " .
 469             "This is turned off by default if you use spice (-vga=qxl).",
 470     },
 471     migrate_speed => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 475         minimum => 0,
 476         default => 0,
 477     },
 478     migrate_downtime => {
 479         optional => 1,
 480         type => 'number',
 481         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 482         minimum => 0,
 483         default => 0.1,
 484     },
 485     cdrom => {
 486         optional => 1,
 487         type => 'string', format => 'pve-qm-ide',
 488         typetext => '<volume>',
 489         description => "This is an alias for option -ide2",
 490     },
 491     cpu => {
 492         optional => 1,
 493         description => "Emulated CPU type.",
 494         type => 'string',
 495         format => $cpu_fmt,
 496     },
 497     parent => get_standard_option('pve-snapshot-name', {
 498         optional => 1,
 499         description => "Parent snapshot name. This is used internally, and should not be modified.",
 500     }),
 501     snaptime => {
 502         optional => 1,
 503         description => "Timestamp for snapshots.",
 504         type => 'integer',
 505         minimum => 0,
 506     },
 507     vmstate => {
 508         optional => 1,
 509         type => 'string', format => 'pve-volume-id',
 510         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 511     },
 512     vmstatestorage => get_standard_option('pve-storage-id', {
 513         description => "Default storage for VM state volumes/files.",
 514         optional => 1,
 515     }),
 516     machine => {
 517         description => "Specific the Qemu machine type.",
 518         type => 'string',
 519         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 520         maxLength => 40,
 521         optional => 1,
 522     },
 523     smbios1 => {
 524         description => "Specify SMBIOS type 1 fields.",
 525         type => 'string', format => 'pve-qm-smbios1',
 526         maxLength => 256,
 527         optional => 1,
 528     },
 529     protection => {
 530         optional => 1,
 531         type => 'boolean',
 532         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 533         default => 0,
 534     },
 535     bios => {
 536         optional => 1,
 537         type => 'string',
 538         enum => [ qw(seabios ovmf) ],
 539         description => "Select BIOS implementation.",
 540         default => 'seabios',
 541     },
 542 };
 543
 544 my $confdesc_cloudinit = {
 545     citype => {
 546         optional => 1,
 547         type => 'string',
 548         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 549         enum => ['configdrive2', 'nocloud'],
 550     },
 551     ciuser => {
 552         optional => 1,
 553         type => 'string',
 554         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 555     },
 556     cipassword => {
 557         optional => 1,
 558         type => 'string',
 559         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 560     },
 561     searchdomain => {
 562         optional => 1,
 563         type => 'string',
 564         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 565     },
 566     nameserver => {
 567         optional => 1,
 568         type => 'string', format => 'address-list',
 569         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 570     },
 571     sshkeys => {
 572         optional => 1,
 573         type => 'string',
 574         format => 'urlencoded',
 575         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 576     },
 577 };
 578
 579 # what about other qemu settings ?
 580 #cpu => 'string',
 581 #machine => 'string',
 582 #fda => 'file',
 583 #fdb => 'file',
 584 #mtdblock => 'file',
 585 #sd => 'file',
 586 #pflash => 'file',
 587 #snapshot => 'bool',
 588 #bootp => 'file',
 589 ##tftp => 'dir',
 590 ##smb => 'dir',
 591 #kernel => 'file',
 592 #append => 'string',
 593 #initrd => 'file',
 594 ##soundhw => 'string',
 595
 596 while (my ($k, $v) = each %$confdesc) {
 597     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 598 }
 599
 600 my $MAX_IDE_DISKS = 4;
 601 my $MAX_SCSI_DISKS = 14;
 602 my $MAX_VIRTIO_DISKS = 16;
 603 my $MAX_SATA_DISKS = 6;
 604 my $MAX_USB_DEVICES = 5;
 605 my $MAX_NETS = 32;
 606 my $MAX_UNUSED_DISKS = 8;
 607 my $MAX_HOSTPCI_DEVICES = 4;
 608 my $MAX_SERIAL_PORTS = 4;
 609 my $MAX_PARALLEL_PORTS = 3;
 610 my $MAX_NUMA = 8;
 611
 612 my $numa_fmt = {
 613     cpus => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "CPUs accessing this NUMA node.",
 617         format_description => "id[-id];...",
 618     },
 619     memory => {
 620         type => "number",
 621         description => "Amount of memory this NUMA node provides.",
 622         optional => 1,
 623     },
 624     hostnodes => {
 625         type => "string",
 626         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 627         description => "Host NUMA nodes to use.",
 628         format_description => "id[-id];...",
 629         optional => 1,
 630     },
 631     policy => {
 632         type => 'string',
 633         enum => [qw(preferred bind interleave)],
 634         description => "NUMA allocation policy.",
 635         optional => 1,
 636     },
 637 };
 638 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 639 my $numadesc = {
 640     optional => 1,
 641     type => 'string', format => $numa_fmt,
 642     description => "NUMA topology.",
 643 };
 644 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 645
 646 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 647     $confdesc->{"numa$i"} = $numadesc;
 648 }
 649
 650 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 651                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 652                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 653 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 654
 655 my $net_fmt_bridge_descr = <<__EOD__;
 656 Bridge to attach the network device to. The Proxmox VE standard bridge
 657 is called 'vmbr0'.
 658
 659 If you do not specify a bridge, we create a kvm user (NATed) network
 660 device, which provides DHCP and DNS services. The following addresses
 661 are used:
 662
 663  10.0.2.2   Gateway
 664  10.0.2.3   DNS Server
 665  10.0.2.4   SMB Server
 666
 667 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 668 __EOD__
 669
 670 my $net_fmt = {
 671     macaddr => {
 672         type => 'string',
 673         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 674         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 675         format_description => "XX:XX:XX:XX:XX:XX",
 676         optional => 1,
 677     },
 678     model => {
 679         type => 'string',
 680         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 681         enum => $nic_model_list,
 682         default_key => 1,
 683     },
 684     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 685     bridge => {
 686         type => 'string',
 687         description => $net_fmt_bridge_descr,
 688         format_description => 'bridge',
 689         optional => 1,
 690     },
 691     queues => {
 692         type => 'integer',
 693         minimum => 0, maximum => 16,
 694         description => 'Number of packet queues to be used on the device.',
 695         optional => 1,
 696     },
 697     rate => {
 698         type => 'number',
 699         minimum => 0,
 700         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 701         optional => 1,
 702     },
 703     tag => {
 704         type => 'integer',
 705         minimum => 1, maximum => 4094,
 706         description => 'VLAN tag to apply to packets on this interface.',
 707         optional => 1,
 708     },
 709     trunks => {
 710         type => 'string',
 711         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 712         description => 'VLAN trunks to pass through this interface.',
 713         format_description => 'vlanid[;vlanid...]',
 714         optional => 1,
 715     },
 716     firewall => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be protected by the firewall.',
 719         optional => 1,
 720     },
 721     link_down => {
 722         type => 'boolean',
 723         description => 'Whether this interface should be disconnected (like pulling the plug).',
 724         optional => 1,
 725     },
 726 };
 727
 728 my $netdesc = {
 729     optional => 1,
 730     type => 'string', format => $net_fmt,
 731     description => "Specify network devices.",
 732 };
 733
 734 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 735
 736 my $ipconfig_fmt = {
 737     ip => {
 738         type => 'string',
 739         format => 'pve-ipv4-config',
 740         format_description => 'IPv4Format/CIDR',
 741         description => 'IPv4 address in CIDR format.',
 742         optional => 1,
 743         default => 'dhcp',
 744     },
 745     gw => {
 746         type => 'string',
 747         format => 'ipv4',
 748         format_description => 'GatewayIPv4',
 749         description => 'Default gateway for IPv4 traffic.',
 750         optional => 1,
 751         requires => 'ip',
 752     },
 753     ip6 => {
 754         type => 'string',
 755         format => 'pve-ipv6-config',
 756         format_description => 'IPv6Format/CIDR',
 757         description => 'IPv6 address in CIDR format.',
 758         optional => 1,
 759         default => 'dhcp',
 760     },
 761     gw6 => {
 762         type => 'string',
 763         format => 'ipv6',
 764         format_description => 'GatewayIPv6',
 765         description => 'Default gateway for IPv6 traffic.',
 766         optional => 1,
 767         requires => 'ip6',
 768     },
 769 };
 770 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 771 my $ipconfigdesc = {
 772     optional => 1,
 773     type => 'string', format => 'pve-qm-ipconfig',
 774     description => <<'EODESCR',
 775 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 776
 777 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 778
 779 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 780 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 781
 782 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 786
 787 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 788     $confdesc->{"net$i"} = $netdesc;
 789     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 790 }
 791
 792 foreach my $key (keys %$confdesc_cloudinit) {
 793     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 797 sub verify_volume_id_or_qm_path {
 798     my ($volid, $noerr) = @_;
 799
 800     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 801         return $volid;
 802     }
 803
 804     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 805     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 806     if ($@) {
 807         return undef if $noerr;
 808         die $@;
 809     }
 810     return $volid;
 811 }
 812
 813 my $drivename_hash;
 814
 815 my %drivedesc_base = (
 816     volume => { alias => 'file' },
 817     file => {
 818         type => 'string',
 819         format => 'pve-volume-id-or-qm-path',
 820         default_key => 1,
 821         format_description => 'volume',
 822         description => "The drive's backing volume.",
 823     },
 824     media => {
 825         type => 'string',
 826         enum => [qw(cdrom disk)],
 827         description => "The drive's media type.",
 828         default => 'disk',
 829         optional => 1
 830     },
 831     cyls => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific cylinder count.",
 834         optional => 1
 835     },
 836     heads => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific head count.",
 839         optional => 1
 840     },
 841     secs => {
 842         type => 'integer',
 843         description => "Force the drive's physical geometry to have a specific sector count.",
 844         optional => 1
 845     },
 846     trans => {
 847         type => 'string',
 848         enum => [qw(none lba auto)],
 849         description => "Force disk geometry bios translation mode.",
 850         optional => 1,
 851     },
 852     snapshot => {
 853         type => 'boolean',
 854         description => "Controls qemu's snapshot mode feature."
 855             . " If activated, changes made to the disk are temporary and will"
 856             . " be discarded when the VM is shutdown.",
 857         optional => 1,
 858     },
 859     cache => {
 860         type => 'string',
 861         enum => [qw(none writethrough writeback unsafe directsync)],
 862         description => "The drive's cache mode",
 863         optional => 1,
 864     },
 865     format => get_standard_option('pve-qm-image-format'),
 866     size => {
 867         type => 'string',
 868         format => 'disk-size',
 869         format_description => 'DiskSize',
 870         description => "Disk size. This is purely informational and has no effect.",
 871         optional => 1,
 872     },
 873     backup => {
 874         type => 'boolean',
 875         description => "Whether the drive should be included when making backups.",
 876         optional => 1,
 877     },
 878     replicate => {
 879         type => 'boolean',
 880         description => 'Whether the drive should considered for replication jobs.',
 881         optional => 1,
 882         default => 1,
 883     },
 884     rerror => {
 885         type => 'string',
 886         enum => [qw(ignore report stop)],
 887         description => 'Read error action.',
 888         optional => 1,
 889     },
 890     werror => {
 891         type => 'string',
 892         enum => [qw(enospc ignore report stop)],
 893         description => 'Write error action.',
 894         optional => 1,
 895     },
 896     aio => {
 897         type => 'string',
 898         enum => [qw(native threads)],
 899         description => 'AIO type to use.',
 900         optional => 1,
 901     },
 902     discard => {
 903         type => 'string',
 904         enum => [qw(ignore on)],
 905         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 906         optional => 1,
 907     },
 908     detect_zeroes => {
 909         type => 'boolean',
 910         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 911         optional => 1,
 912     },
 913     serial => {
 914         type => 'string',
 915         format => 'urlencoded',
 916         format_description => 'serial',
 917         maxLength => 20*3, # *3 since it's %xx url enoded
 918         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 919         optional => 1,
 920     },
 921     shared => {
 922         type => 'boolean',
 923         description => 'Mark this locally-managed volume as available on all nodes',
 924         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 925         optional => 1,
 926         default => 0,
 927     }
 928 );
 929
 930 my %iothread_fmt = ( iothread => {
 931         type => 'boolean',
 932         description => "Whether to use iothreads for this drive",
 933         optional => 1,
 934 });
 935
 936 my %model_fmt = (
 937     model => {
 938         type => 'string',
 939         format => 'urlencoded',
 940         format_description => 'model',
 941         maxLength => 40*3, # *3 since it's %xx url enoded
 942         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 943         optional => 1,
 944     },
 945 );
 946
 947 my %queues_fmt = (
 948     queues => {
 949         type => 'integer',
 950         description => "Number of queues.",
 951         minimum => 2,
 952         optional => 1
 953     }
 954 );
 955
 956 my %scsiblock_fmt = (
 957     scsiblock => {
 958         type => 'boolean',
 959         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 960         optional => 1,
 961         default => 0,
 962     },
 963 );
 964
 965 my $add_throttle_desc = sub {
 966     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 967     my $d = {
 968         type => $type,
 969         format_description => $unit,
 970         description => "Maximum $what in $longunit.",
 971         optional => 1,
 972     };
 973     $d->{minimum} = $minimum if defined($minimum);
 974     $drivedesc_base{$key} = $d;
 975 };
 976 # throughput: (leaky bucket)
 977 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 979 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 980 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 982 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 983 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 984 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 985 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 986
 987 # pools: (pool of IO before throttling starts taking effect)
 988 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 990 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 991 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 992 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 993 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 994
 995 # burst lengths
 996 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 998 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1001 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1002
1003 # legacy support
1004 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1005 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1006 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1007 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1008
1009 my $ide_fmt = {
1010     %drivedesc_base,
1011     %model_fmt,
1012 };
1013 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1014
1015 my $idedesc = {
1016     optional => 1,
1017     type => 'string', format => $ide_fmt,
1018     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1019 };
1020 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1021
1022 my $scsi_fmt = {
1023     %drivedesc_base,
1024     %iothread_fmt,
1025     %queues_fmt,
1026     %scsiblock_fmt,
1027 };
1028 my $scsidesc = {
1029     optional => 1,
1030     type => 'string', format => $scsi_fmt,
1031     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1034
1035 my $sata_fmt = {
1036     %drivedesc_base,
1037 };
1038 my $satadesc = {
1039     optional => 1,
1040     type => 'string', format => $sata_fmt,
1041     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1044
1045 my $virtio_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048 };
1049 my $virtiodesc = {
1050     optional => 1,
1051     type => 'string', format => $virtio_fmt,
1052     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1053 };
1054 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1055
1056 my $alldrive_fmt = {
1057     %drivedesc_base,
1058     %iothread_fmt,
1059     %model_fmt,
1060     %queues_fmt,
1061     %scsiblock_fmt,
1062 };
1063
1064 my $efidisk_fmt = {
1065     volume => { alias => 'file' },
1066     file => {
1067         type => 'string',
1068         format => 'pve-volume-id-or-qm-path',
1069         default_key => 1,
1070         format_description => 'volume',
1071         description => "The drive's backing volume.",
1072     },
1073     format => get_standard_option('pve-qm-image-format'),
1074     size => {
1075         type => 'string',
1076         format => 'disk-size',
1077         format_description => 'DiskSize',
1078         description => "Disk size. This is purely informational and has no effect.",
1079         optional => 1,
1080     },
1081 };
1082
1083 my $efidisk_desc = {
1084     optional => 1,
1085     type => 'string', format => $efidisk_fmt,
1086     description => "Configure a Disk for storing EFI vars",
1087 };
1088
1089 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1090
1091 my $usb_fmt = {
1092     host => {
1093         default_key => 1,
1094         type => 'string', format => 'pve-qm-usb-device',
1095         format_description => 'HOSTUSBDEVICE|spice',
1096         description => <<EODESCR,
1097 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1098
1099  'bus-port(.port)*' (decimal numbers) or
1100  'vendor_id:product_id' (hexadeciaml numbers) or
1101  'spice'
1102
1103 You can use the 'lsusb -t' command to list existing usb devices.
1104
1105 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1106
1107 The value 'spice' can be used to add a usb redirection devices for spice.
1108 EODESCR
1109     },
1110     usb3 => {
1111         optional => 1,
1112         type => 'boolean',
1113         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1114         default => 0,
1115     },
1116 };
1117
1118 my $usbdesc = {
1119     optional => 1,
1120     type => 'string', format => $usb_fmt,
1121     description => "Configure an USB device (n is 0 to 4).",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1124
1125 # NOTE: the match-groups of this regex are used in parse_hostpci
1126 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1127 my $hostpci_fmt = {
1128     host => {
1129         default_key => 1,
1130         type => 'string',
1131         pattern => qr/$PCIRE(;$PCIRE)*/,
1132         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1133         description => <<EODESCR,
1134 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1135 of PCI virtual functions of the host. HOSTPCIID syntax is:
1136
1137 'bus:dev.func' (hexadecimal numbers)
1138
1139 You can us the 'lspci' command to list existing PCI devices.
1140 EODESCR
1141     },
1142     rombar => {
1143         type => 'boolean',
1144         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1145         optional => 1,
1146         default => 1,
1147     },
1148     romfile => {
1149         type => 'string',
1150         pattern => '[^,;]+',
1151         format_description => 'string',
1152         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1153         optional => 1,
1154     },
1155     pcie => {
1156         type => 'boolean',
1157         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1158         optional => 1,
1159         default => 0,
1160     },
1161     'x-vga' => {
1162         type => 'boolean',
1163         description =>  "Enable vfio-vga device support.",
1164         optional => 1,
1165         default => 0,
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1169
1170 my $hostpcidesc = {
1171         optional => 1,
1172         type => 'string', format => 'pve-qm-hostpci',
1173         description => "Map host PCI devices into guest.",
1174         verbose_description =>  <<EODESCR,
1175 Map host PCI devices into guest.
1176
1177 NOTE: This option allows direct access to host hardware. So it is no longer
1178 possible to migrate such machines - use with special care.
1179
1180 CAUTION: Experimental! User reported problems with this option.
1181 EODESCR
1182 };
1183 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1184
1185 my $serialdesc = {
1186         optional => 1,
1187         type => 'string',
1188         pattern => '(/dev/.+|socket)',
1189         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1190         verbose_description =>  <<EODESCR,
1191 Create a serial device inside the VM (n is 0 to 3), and pass through a
1192 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1193 host side (use 'qm terminal' to open a terminal connection).
1194
1195 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1196
1197 CAUTION: Experimental! User reported problems with this option.
1198 EODESCR
1199 };
1200
1201 my $paralleldesc= {
1202         optional => 1,
1203         type => 'string',
1204         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1205         description =>  "Map host parallel devices (n is 0 to 2).",
1206         verbose_description =>  <<EODESCR,
1207 Map host parallel devices (n is 0 to 2).
1208
1209 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1210
1211 CAUTION: Experimental! User reported problems with this option.
1212 EODESCR
1213 };
1214
1215 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1216     $confdesc->{"parallel$i"} = $paralleldesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1220     $confdesc->{"serial$i"} = $serialdesc;
1221 }
1222
1223 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1224     $confdesc->{"hostpci$i"} = $hostpcidesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1228     $drivename_hash->{"ide$i"} = 1;
1229     $confdesc->{"ide$i"} = $idedesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1233     $drivename_hash->{"sata$i"} = 1;
1234     $confdesc->{"sata$i"} = $satadesc;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1238     $drivename_hash->{"scsi$i"} = 1;
1239     $confdesc->{"scsi$i"} = $scsidesc ;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1243     $drivename_hash->{"virtio$i"} = 1;
1244     $confdesc->{"virtio$i"} = $virtiodesc;
1245 }
1246
1247 $drivename_hash->{efidisk0} = 1;
1248 $confdesc->{efidisk0} = $efidisk_desc;
1249
1250 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1251     $confdesc->{"usb$i"} = $usbdesc;
1252 }
1253
1254 my $unuseddesc = {
1255     optional => 1,
1256     type => 'string', format => 'pve-volume-id',
1257     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1258 };
1259
1260 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1261     $confdesc->{"unused$i"} = $unuseddesc;
1262 }
1263
1264 my $kvm_api_version = 0;
1265
1266 sub kvm_version {
1267
1268     return $kvm_api_version if $kvm_api_version;
1269
1270     my $fh = IO::File->new("</dev/kvm") ||
1271         return 0;
1272
1273     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1274         $kvm_api_version = $v;
1275     }
1276
1277     $fh->close();
1278
1279     return  $kvm_api_version;
1280 }
1281
1282 my $kvm_user_version;
1283
1284 sub kvm_user_version {
1285
1286     return $kvm_user_version if $kvm_user_version;
1287
1288     $kvm_user_version = 'unknown';
1289
1290     my $code = sub {
1291         my $line = shift;
1292         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1293             $kvm_user_version = $2;
1294         }
1295     };
1296
1297     eval { run_command("kvm -version", outfunc => $code); };
1298     warn $@ if $@;
1299
1300     return $kvm_user_version;
1301
1302 }
1303
1304 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1305
1306 sub valid_drive_names {
1307     # order is important - used to autoselect boot disk
1308     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1309             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1310             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1311             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1312             'efidisk0');
1313 }
1314
1315 sub is_valid_drivename {
1316     my $dev = shift;
1317
1318     return defined($drivename_hash->{$dev});
1319 }
1320
1321 sub option_exists {
1322     my $key = shift;
1323     return defined($confdesc->{$key});
1324 }
1325
1326 sub nic_models {
1327     return $nic_model_list;
1328 }
1329
1330 sub os_list_description {
1331
1332     return {
1333         other => 'Other',
1334         wxp => 'Windows XP',
1335         w2k => 'Windows 2000',
1336         w2k3 =>, 'Windows 2003',
1337         w2k8 => 'Windows 2008',
1338         wvista => 'Windows Vista',
1339         win7 => 'Windows 7',
1340         win8 => 'Windows 8/2012',
1341         win10 => 'Windows 10/2016',
1342         l24 => 'Linux 2.4',
1343         l26 => 'Linux 2.6',
1344     };
1345 }
1346
1347 my $cdrom_path;
1348
1349 sub get_cdrom_path {
1350
1351     return  $cdrom_path if $cdrom_path;
1352
1353     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1354     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1355     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1356 }
1357
1358 sub get_iso_path {
1359     my ($storecfg, $vmid, $cdrom) = @_;
1360
1361     if ($cdrom eq 'cdrom') {
1362         return get_cdrom_path();
1363     } elsif ($cdrom eq 'none') {
1364         return '';
1365     } elsif ($cdrom =~ m|^/|) {
1366         return $cdrom;
1367     } else {
1368         return PVE::Storage::path($storecfg, $cdrom);
1369     }
1370 }
1371
1372 # try to convert old style file names to volume IDs
1373 sub filename_to_volume_id {
1374     my ($vmid, $file, $media) = @_;
1375
1376      if (!($file eq 'none' || $file eq 'cdrom' ||
1377           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1378
1379         return undef if $file =~ m|/|;
1380
1381         if ($media && $media eq 'cdrom') {
1382             $file = "local:iso/$file";
1383         } else {
1384             $file = "local:$vmid/$file";
1385         }
1386     }
1387
1388     return $file;
1389 }
1390
1391 sub verify_media_type {
1392     my ($opt, $vtype, $media) = @_;
1393
1394     return if !$media;
1395
1396     my $etype;
1397     if ($media eq 'disk') {
1398         $etype = 'images';
1399     } elsif ($media eq 'cdrom') {
1400         $etype = 'iso';
1401     } else {
1402         die "internal error";
1403     }
1404
1405     return if ($vtype eq $etype);
1406
1407     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1408 }
1409
1410 sub cleanup_drive_path {
1411     my ($opt, $storecfg, $drive) = @_;
1412
1413     # try to convert filesystem paths to volume IDs
1414
1415     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1416         ($drive->{file} !~ m|^/dev/.+|) &&
1417         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1418         ($drive->{file} !~ m/^\d+$/)) {
1419         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1420         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1421         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1422         verify_media_type($opt, $vtype, $drive->{media});
1423         $drive->{file} = $volid;
1424     }
1425
1426     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1427 }
1428
1429 sub parse_hotplug_features {
1430     my ($data) = @_;
1431
1432     my $res = {};
1433
1434     return $res if $data eq '0';
1435
1436     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1437
1438     foreach my $feature (PVE::Tools::split_list($data)) {
1439         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1440             $res->{$1} = 1;
1441         } else {
1442             die "invalid hotplug feature '$feature'\n";
1443         }
1444     }
1445     return $res;
1446 }
1447
1448 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1449 sub pve_verify_hotplug_features {
1450     my ($value, $noerr) = @_;
1451
1452     return $value if parse_hotplug_features($value);
1453
1454     return undef if $noerr;
1455
1456     die "unable to parse hotplug option\n";
1457 }
1458
1459 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1460 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1461 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1462 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1463 #        [,iothread=on][,serial=serial][,model=model]
1464
1465 sub parse_drive {
1466     my ($key, $data) = @_;
1467
1468     my ($interface, $index);
1469
1470     if ($key =~ m/^([^\d]+)(\d+)$/) {
1471         $interface = $1;
1472         $index = $2;
1473     } else {
1474         return undef;
1475     }
1476
1477     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1478                                      : $confdesc->{$key}->{format};
1479     if (!$desc) {
1480         warn "invalid drive key: $key\n";
1481         return undef;
1482     }
1483     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1484     return undef if !$res;
1485     $res->{interface} = $interface;
1486     $res->{index} = $index;
1487
1488     my $error = 0;
1489     foreach my $opt (qw(bps bps_rd bps_wr)) {
1490         if (my $bps = defined(delete $res->{$opt})) {
1491             if (defined($res->{"m$opt"})) {
1492                 warn "both $opt and m$opt specified\n";
1493                 ++$error;
1494                 next;
1495             }
1496             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1497         }
1498     }
1499
1500     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1501     for my $requirement (
1502         [mbps_max => 'mbps'],
1503         [mbps_rd_max => 'mbps_rd'],
1504         [mbps_wr_max => 'mbps_wr'],
1505         [miops_max => 'miops'],
1506         [miops_rd_max => 'miops_rd'],
1507         [miops_wr_max => 'miops_wr'],
1508         [bps_max_length => 'mbps_max'],
1509         [bps_rd_max_length => 'mbps_rd_max'],
1510         [bps_wr_max_length => 'mbps_wr_max'],
1511         [iops_max_length => 'iops_max'],
1512         [iops_rd_max_length => 'iops_rd_max'],
1513         [iops_wr_max_length => 'iops_wr_max']) {
1514         my ($option, $requires) = @$requirement;
1515         if ($res->{$option} && !$res->{$requires}) {
1516             warn "$option requires $requires\n";
1517             ++$error;
1518         }
1519     }
1520
1521     return undef if $error;
1522
1523     return undef if $res->{mbps_rd} && $res->{mbps};
1524     return undef if $res->{mbps_wr} && $res->{mbps};
1525     return undef if $res->{iops_rd} && $res->{iops};
1526     return undef if $res->{iops_wr} && $res->{iops};
1527
1528     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1529         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1530         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1531         return undef if $res->{interface} eq 'virtio';
1532     }
1533
1534     if (my $size = $res->{size}) {
1535         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub print_drive {
1542     my ($vmid, $drive) = @_;
1543     my $data = { %$drive };
1544     delete $data->{$_} for qw(index interface);
1545     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1546 }
1547
1548 sub scsi_inquiry {
1549     my($fh, $noerr) = @_;
1550
1551     my $SG_IO = 0x2285;
1552     my $SG_GET_VERSION_NUM = 0x2282;
1553
1554     my $versionbuf = "\x00" x 8;
1555     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1556     if (!$ret) {
1557         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1558         return undef;
1559     }
1560     my $version = unpack("I", $versionbuf);
1561     if ($version < 30000) {
1562         die "scsi generic interface too old\n"  if !$noerr;
1563         return undef;
1564     }
1565
1566     my $buf = "\x00" x 36;
1567     my $sensebuf = "\x00" x 8;
1568     my $cmd = pack("C x3 C x1", 0x12, 36);
1569
1570     # see /usr/include/scsi/sg.h
1571     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1572
1573     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1574                       length($sensebuf), 0, length($buf), $buf,
1575                       $cmd, $sensebuf, 6000);
1576
1577     $ret = ioctl($fh, $SG_IO, $packet);
1578     if (!$ret) {
1579         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1580         return undef;
1581     }
1582
1583     my @res = unpack($sg_io_hdr_t, $packet);
1584     if ($res[17] || $res[18]) {
1585         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1586         return undef;
1587     }
1588
1589     my $res = {};
1590     (my $byte0, my $byte1, $res->{vendor},
1591      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1592
1593     $res->{removable} = $byte1 & 128 ? 1 : 0;
1594     $res->{type} = $byte0 & 31;
1595
1596     return $res;
1597 }
1598
1599 sub path_is_scsi {
1600     my ($path) = @_;
1601
1602     my $fh = IO::File->new("+<$path") || return undef;
1603     my $res = scsi_inquiry($fh, 1);
1604     close($fh);
1605
1606     return $res;
1607 }
1608
1609 sub machine_type_is_q35 {
1610     my ($conf) = @_;
1611
1612     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1613 }
1614
1615 sub print_tabletdevice_full {
1616     my ($conf) = @_;
1617
1618     my $q35 = machine_type_is_q35($conf);
1619
1620     # we use uhci for old VMs because tablet driver was buggy in older qemu
1621     my $usbbus = $q35 ? "ehci" : "uhci";
1622
1623     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1624 }
1625
1626 sub print_drivedevice_full {
1627     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1628
1629     my $device = '';
1630     my $maxdev = 0;
1631
1632     if ($drive->{interface} eq 'virtio') {
1633         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1634         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1635         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1636     } elsif ($drive->{interface} eq 'scsi') {
1637
1638         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1639         my $unit = $drive->{index} % $maxdev;
1640         my $devicetype = 'hd';
1641         my $path = '';
1642         if (drive_is_cdrom($drive)) {
1643             $devicetype = 'cd';
1644         } else {
1645             if ($drive->{file} =~ m|^/|) {
1646                 $path = $drive->{file};
1647                 if (my $info = path_is_scsi($path)) {
1648                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1649                         $devicetype = 'block';
1650                     } elsif ($info->{type} == 1) { # tape
1651                         $devicetype = 'generic';
1652                     }
1653                 }
1654             } else {
1655                  $path = PVE::Storage::path($storecfg, $drive->{file});
1656             }
1657
1658             if($path =~ m/^iscsi\:\/\//){
1659                 $devicetype = 'generic';
1660             }
1661         }
1662
1663         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         } else {
1666             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1667         }
1668
1669     } elsif ($drive->{interface} eq 'ide'){
1670         $maxdev = 2;
1671         my $controller = int($drive->{index} / $maxdev);
1672         my $unit = $drive->{index} % $maxdev;
1673         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1674
1675         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1676         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1677             $model = URI::Escape::uri_unescape($model);
1678             $device .= ",model=$model";
1679         }
1680     } elsif ($drive->{interface} eq 'sata'){
1681         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1682         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1683         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1684     } elsif ($drive->{interface} eq 'usb') {
1685         die "implement me";
1686         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1687     } else {
1688         die "unsupported interface type";
1689     }
1690
1691     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1692
1693     if (my $serial = $drive->{serial}) {
1694         $serial = URI::Escape::uri_unescape($serial);
1695         $device .= ",serial=$serial";
1696     }
1697
1698
1699     return $device;
1700 }
1701
1702 sub get_initiator_name {
1703     my $initiator;
1704
1705     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1706     while (defined(my $line = <$fh>)) {
1707         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1708         $initiator = $1;
1709         last;
1710     }
1711     $fh->close();
1712
1713     return $initiator;
1714 }
1715
1716 sub print_drive_full {
1717     my ($storecfg, $vmid, $drive) = @_;
1718
1719     my $path;
1720     my $volid = $drive->{file};
1721     my $format;
1722
1723     if (drive_is_cdrom($drive)) {
1724         $path = get_iso_path($storecfg, $vmid, $volid);
1725     } else {
1726         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1727         if ($storeid) {
1728             $path = PVE::Storage::path($storecfg, $volid);
1729             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1730             $format = qemu_img_format($scfg, $volname);
1731         } else {
1732             $path = $volid;
1733             $format = "raw";
1734         }
1735    }
1736
1737     my $opts = '';
1738     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1739     foreach my $o (@qemu_drive_options) {
1740         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1741     }
1742
1743     # snapshot only accepts on|off
1744     if (defined($drive->{snapshot})) {
1745         my $v = $drive->{snapshot} ? 'on' : 'off';
1746         $opts .= ",snapshot=$v";
1747     }
1748
1749     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1750         my ($dir, $qmpname) = @$type;
1751         if (my $v = $drive->{"mbps$dir"}) {
1752             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1753         }
1754         if (my $v = $drive->{"mbps${dir}_max"}) {
1755             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1756         }
1757         if (my $v = $drive->{"bps${dir}_max_length"}) {
1758             $opts .= ",throttling.bps$qmpname-max-length=$v";
1759         }
1760         if (my $v = $drive->{"iops${dir}"}) {
1761             $opts .= ",throttling.iops$qmpname=$v";
1762         }
1763         if (my $v = $drive->{"iops${dir}_max"}) {
1764             $opts .= ",throttling.iops$qmpname-max=$v";
1765         }
1766         if (my $v = $drive->{"iops${dir}_max_length"}) {
1767             $opts .= ",throttling.iops$qmpname-max-length=$v";
1768         }
1769     }
1770
1771     $opts .= ",format=$format" if $format && !$drive->{format};
1772
1773     my $cache_direct = 0;
1774
1775     if (my $cache = $drive->{cache}) {
1776         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1777     } elsif (!drive_is_cdrom($drive)) {
1778         $opts .= ",cache=none";
1779         $cache_direct = 1;
1780     }
1781
1782     # aio native works only with O_DIRECT
1783     if (!$drive->{aio}) {
1784         if($cache_direct) {
1785             $opts .= ",aio=native";
1786         } else {
1787             $opts .= ",aio=threads";
1788         }
1789     }
1790
1791     if (!drive_is_cdrom($drive)) {
1792         my $detectzeroes;
1793         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1794             $detectzeroes = 'off';
1795         } elsif ($drive->{discard}) {
1796             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1797         } else {
1798             # This used to be our default with discard not being specified:
1799             $detectzeroes = 'on';
1800         }
1801         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1802     }
1803
1804     my $pathinfo = $path ? "file=$path," : '';
1805
1806     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1807 }
1808
1809 sub print_netdevice_full {
1810     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1811
1812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1813
1814     my $device = $net->{model};
1815     if ($net->{model} eq 'virtio') {
1816          $device = 'virtio-net-pci';
1817      };
1818
1819     my $pciaddr = print_pci_addr("$netid", $bridges);
1820     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1821     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1822         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1823         my $vectors = $net->{queues} * 2 + 2;
1824         $tmpstr .= ",vectors=$vectors,mq=on";
1825     }
1826     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1827
1828     if ($use_old_bios_files) {
1829         my $romfile;
1830         if ($device eq 'virtio-net-pci') {
1831             $romfile = 'pxe-virtio.rom';
1832         } elsif ($device eq 'e1000') {
1833             $romfile = 'pxe-e1000.rom';
1834         } elsif ($device eq 'ne2k') {
1835             $romfile = 'pxe-ne2k_pci.rom';
1836         } elsif ($device eq 'pcnet') {
1837             $romfile = 'pxe-pcnet.rom';
1838         } elsif ($device eq 'rtl8139') {
1839             $romfile = 'pxe-rtl8139.rom';
1840         }
1841         $tmpstr .= ",romfile=$romfile" if $romfile;
1842     }
1843
1844     return $tmpstr;
1845 }
1846
1847 sub print_netdev_full {
1848     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1849
1850     my $i = '';
1851     if ($netid =~ m/^net(\d+)$/) {
1852         $i = int($1);
1853     }
1854
1855     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1856
1857     my $ifname = "tap${vmid}i$i";
1858
1859     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1860     die "interface name '$ifname' is too long (max 15 character)\n"
1861         if length($ifname) >= 16;
1862
1863     my $vhostparam = '';
1864     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1865
1866     my $vmname = $conf->{name} || "vm$vmid";
1867
1868     my $netdev = "";
1869     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1870
1871     if ($net->{bridge}) {
1872         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1873     } else {
1874         $netdev = "type=user,id=$netid,hostname=$vmname";
1875     }
1876
1877     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1878
1879     return $netdev;
1880 }
1881
1882
1883 sub print_cpu_device {
1884     my ($conf, $id) = @_;
1885
1886     my $kvm = $conf->{kvm} // 1;
1887     my $cpu = $kvm ? "kvm64" : "qemu64";
1888     if (my $cputype = $conf->{cpu}) {
1889         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1890             or die "Cannot parse cpu description: $cputype\n";
1891         $cpu = $cpuconf->{cputype};
1892     }
1893
1894     my $cores = $conf->{cores} || 1;
1895
1896     my $current_core = ($id - 1) % $cores;
1897     my $current_socket = int(($id - 1 - $current_core)/$cores);
1898
1899     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1900 }
1901
1902 sub drive_is_cloudinit {
1903     my ($drive) = @_;
1904     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1905 }
1906
1907 sub drive_is_cdrom {
1908     my ($drive, $exclude_cloudinit) = @_;
1909
1910     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1911
1912     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1913
1914 }
1915
1916 sub parse_number_sets {
1917     my ($set) = @_;
1918     my $res = [];
1919     foreach my $part (split(/;/, $set)) {
1920         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1921             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1922             push @$res, [ $1, $2 ];
1923         } else {
1924             die "invalid range: $part\n";
1925         }
1926     }
1927     return $res;
1928 }
1929
1930 sub parse_numa {
1931     my ($data) = @_;
1932
1933     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1934     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1935     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1936     return $res;
1937 }
1938
1939 sub parse_hostpci {
1940     my ($value) = @_;
1941
1942     return undef if !$value;
1943
1944     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1945
1946     my @idlist = split(/;/, $res->{host});
1947     delete $res->{host};
1948     foreach my $id (@idlist) {
1949         if ($id =~ /^$PCIRE$/) {
1950             if (defined($2)) {
1951                 push @{$res->{pciid}}, { id => $1, function => $2 };
1952             } else {
1953                 my $pcidevices = lspci($1);
1954                 $res->{pciid} = $pcidevices->{$1};
1955             }
1956         } else {
1957             # should have been caught by parse_property_string already
1958             die "failed to parse PCI id: $id\n";
1959         }
1960     }
1961     return $res;
1962 }
1963
1964 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1965 sub parse_net {
1966     my ($data) = @_;
1967
1968     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1969     if ($@) {
1970         warn $@;
1971         return undef;
1972     }
1973     if (!defined($res->{macaddr})) {
1974         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1975         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1976     }
1977     return $res;
1978 }
1979
1980 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1981 sub parse_ipconfig {
1982     my ($data) = @_;
1983
1984     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return undef;
1988     }
1989
1990     if ($res->{gw} && !$res->{ip}) {
1991         warn 'gateway specified without specifying an IP address';
1992         return undef;
1993     }
1994     if ($res->{gw6} && !$res->{ip6}) {
1995         warn 'IPv6 gateway specified without specifying an IPv6 address';
1996         return undef;
1997     }
1998     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1999         warn 'gateway specified together with DHCP';
2000         return undef;
2001     }
2002     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2003         # gw6 + auto/dhcp
2004         warn "IPv6 gateway specified together with $res->{ip6} address";
2005         return undef;
2006     }
2007
2008     if (!$res->{ip} && !$res->{ip6}) {
2009         return { ip => 'dhcp', ip6 => 'dhcp' };
2010     }
2011
2012     return $res;
2013 }
2014
2015 sub print_net {
2016     my $net = shift;
2017
2018     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2019 }
2020
2021 sub add_random_macs {
2022     my ($settings) = @_;
2023
2024     foreach my $opt (keys %$settings) {
2025         next if $opt !~ m/^net(\d+)$/;
2026         my $net = parse_net($settings->{$opt});
2027         next if !$net;
2028         $settings->{$opt} = print_net($net);
2029     }
2030 }
2031
2032 sub vm_is_volid_owner {
2033     my ($storecfg, $vmid, $volid) = @_;
2034
2035     if ($volid !~  m|^/|) {
2036         my ($path, $owner);
2037         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2038         if ($owner && ($owner == $vmid)) {
2039             return 1;
2040         }
2041     }
2042
2043     return undef;
2044 }
2045
2046 sub split_flagged_list {
2047     my $text = shift || '';
2048     $text =~ s/[,;]/ /g;
2049     $text =~ s/^\s+//;
2050     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2051 }
2052
2053 sub join_flagged_list {
2054     my ($how, $lst) = @_;
2055     join $how, map { $lst->{$_} . $_ } keys %$lst;
2056 }
2057
2058 sub vmconfig_delete_pending_option {
2059     my ($conf, $key, $force) = @_;
2060
2061     delete $conf->{pending}->{$key};
2062     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2063     $pending_delete_hash->{$key} = $force ? '!' : '';
2064     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2065 }
2066
2067 sub vmconfig_undelete_pending_option {
2068     my ($conf, $key) = @_;
2069
2070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2071     delete $pending_delete_hash->{$key};
2072
2073     if (%$pending_delete_hash) {
2074         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2075     } else {
2076         delete $conf->{pending}->{delete};
2077     }
2078 }
2079
2080 sub vmconfig_register_unused_drive {
2081     my ($storecfg, $vmid, $conf, $drive) = @_;
2082
2083     if (drive_is_cloudinit($drive)) {
2084         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2085         warn $@ if $@;
2086     } elsif (!drive_is_cdrom($drive)) {
2087         my $volid = $drive->{file};
2088         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2089             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2090         }
2091     }
2092 }
2093
2094 sub vmconfig_cleanup_pending {
2095     my ($conf) = @_;
2096
2097     # remove pending changes when nothing changed
2098     my $changes;
2099     foreach my $opt (keys %{$conf->{pending}}) {
2100         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2101             $changes = 1;
2102             delete $conf->{pending}->{$opt};
2103         }
2104     }
2105
2106     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2107     my $pending_delete_hash = {};
2108     while (my ($opt, $force) = each %$current_delete_hash) {
2109         if (defined($conf->{$opt})) {
2110             $pending_delete_hash->{$opt} = $force;
2111         } else {
2112             $changes = 1;
2113         }
2114     }
2115
2116     if (%$pending_delete_hash) {
2117         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2118     } else {
2119         delete $conf->{pending}->{delete};
2120     }
2121
2122     return $changes;
2123 }
2124
2125 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2126 my $smbios1_fmt = {
2127     uuid => {
2128         type => 'string',
2129         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2130         format_description => 'UUID',
2131         description => "Set SMBIOS1 UUID.",
2132         optional => 1,
2133     },
2134     version => {
2135         type => 'string',
2136         pattern => '\S+',
2137         format_description => 'string',
2138         description => "Set SMBIOS1 version.",
2139         optional => 1,
2140     },
2141     serial => {
2142         type => 'string',
2143         pattern => '\S+',
2144         format_description => 'string',
2145         description => "Set SMBIOS1 serial number.",
2146         optional => 1,
2147     },
2148     manufacturer => {
2149         type => 'string',
2150         pattern => '\S+',
2151         format_description => 'string',
2152         description => "Set SMBIOS1 manufacturer.",
2153         optional => 1,
2154     },
2155     product => {
2156         type => 'string',
2157         pattern => '\S+',
2158         format_description => 'string',
2159         description => "Set SMBIOS1 product ID.",
2160         optional => 1,
2161     },
2162     sku => {
2163         type => 'string',
2164         pattern => '\S+',
2165         format_description => 'string',
2166         description => "Set SMBIOS1 SKU string.",
2167         optional => 1,
2168     },
2169     family => {
2170         type => 'string',
2171         pattern => '\S+',
2172         format_description => 'string',
2173         description => "Set SMBIOS1 family string.",
2174         optional => 1,
2175     },
2176 };
2177
2178 sub parse_smbios1 {
2179     my ($data) = @_;
2180
2181     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub print_smbios1 {
2187     my ($smbios1) = @_;
2188     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2192
2193 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2194 sub verify_bootdisk {
2195     my ($value, $noerr) = @_;
2196
2197     return $value if is_valid_drivename($value);
2198
2199     return undef if $noerr;
2200
2201     die "invalid boot disk '$value'\n";
2202 }
2203
2204 sub parse_watchdog {
2205     my ($value) = @_;
2206
2207     return undef if !$value;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2210     warn $@ if $@;
2211     return $res;
2212 }
2213
2214 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2215 sub verify_usb_device {
2216     my ($value, $noerr) = @_;
2217
2218     return $value if parse_usb_device($value);
2219
2220     return undef if $noerr;
2221
2222     die "unable to parse usb device\n";
2223 }
2224
2225 # add JSON properties for create and set function
2226 sub json_config_properties {
2227     my $prop = shift;
2228
2229     foreach my $opt (keys %$confdesc) {
2230         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2231         $prop->{$opt} = $confdesc->{$opt};
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # return copy of $confdesc_cloudinit to generate documentation
2238 sub cloudinit_config_properties {
2239
2240     return dclone($confdesc_cloudinit);
2241 }
2242
2243 sub check_type {
2244     my ($key, $value) = @_;
2245
2246     die "unknown setting '$key'\n" if !$confdesc->{$key};
2247
2248     my $type = $confdesc->{$key}->{type};
2249
2250     if (!defined($value)) {
2251         die "got undefined value\n";
2252     }
2253
2254     if ($value =~ m/[\n\r]/) {
2255         die "property contains a line feed\n";
2256     }
2257
2258     if ($type eq 'boolean') {
2259         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2260         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2261         die "type check ('boolean') failed - got '$value'\n";
2262     } elsif ($type eq 'integer') {
2263         return int($1) if $value =~ m/^(\d+)$/;
2264         die "type check ('integer') failed - got '$value'\n";
2265     } elsif ($type eq 'number') {
2266         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2267         die "type check ('number') failed - got '$value'\n";
2268     } elsif ($type eq 'string') {
2269         if (my $fmt = $confdesc->{$key}->{format}) {
2270             PVE::JSONSchema::check_format($fmt, $value);
2271             return $value;
2272         }
2273         $value =~ s/^\"(.*)\"$/$1/;
2274         return $value;
2275     } else {
2276         die "internal error"
2277     }
2278 }
2279
2280 sub check_iommu_support{
2281     #fixme : need to check IOMMU support
2282     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2283
2284     my $iommu=1;
2285     return $iommu;
2286
2287 }
2288
2289 sub touch_config {
2290     my ($vmid) = @_;
2291
2292     my $conf = PVE::QemuConfig->config_file($vmid);
2293     utime undef, undef, $conf;
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2298
2299     my $conffile = PVE::QemuConfig->config_file($vmid);
2300
2301     my $conf = PVE::QemuConfig->load_config($vmid);
2302
2303     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2304
2305     if ($conf->{template}) {
2306         # check if any base image is still used by a linked clone
2307         foreach_drive($conf, sub {
2308                 my ($ds, $drive) = @_;
2309
2310                 return if drive_is_cdrom($drive);
2311
2312                 my $volid = $drive->{file};
2313
2314                 return if !$volid || $volid =~ m|^/|;
2315
2316                 die "base volume '$volid' is still in use by linked cloned\n"
2317                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2318
2319         });
2320     }
2321
2322     # only remove disks owned by this VM
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         return if drive_is_cdrom($drive, 1);
2327
2328         my $volid = $drive->{file};
2329
2330         return if !$volid || $volid =~ m|^/|;
2331
2332         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2333         return if !$path || !$owner || ($owner != $vmid);
2334
2335         eval {
2336             PVE::Storage::vdisk_free($storecfg, $volid);
2337         };
2338         warn "Could not remove disk '$volid', check manually: $@" if $@;
2339
2340     });
2341
2342     if ($keep_empty_config) {
2343         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2344     } else {
2345         unlink $conffile;
2346     }
2347
2348     # also remove unused disk
2349     eval {
2350         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2351
2352         eval {
2353             PVE::Storage::foreach_volid($dl, sub {
2354                 my ($volid, $sid, $volname, $d) = @_;
2355                 PVE::Storage::vdisk_free($storecfg, $volid);
2356             });
2357         };
2358         warn $@ if $@;
2359
2360     };
2361     warn $@ if $@;
2362 }
2363
2364 sub parse_vm_config {
2365     my ($filename, $raw) = @_;
2366
2367     return undef if !defined($raw);
2368
2369     my $res = {
2370         digest => Digest::SHA::sha1_hex($raw),
2371         snapshots => {},
2372         pending => {},
2373     };
2374
2375     $filename =~ m|/qemu-server/(\d+)\.conf$|
2376         || die "got strange filename '$filename'";
2377
2378     my $vmid = $1;
2379
2380     my $conf = $res;
2381     my $descr;
2382     my $section = '';
2383
2384     my @lines = split(/\n/, $raw);
2385     foreach my $line (@lines) {
2386         next if $line =~ m/^\s*$/;
2387
2388         if ($line =~ m/^\[PENDING\]\s*$/i) {
2389             $section = 'pending';
2390             if (defined($descr)) {
2391                 $descr =~ s/\s+$//;
2392                 $conf->{description} = $descr;
2393             }
2394             $descr = undef;
2395             $conf = $res->{$section} = {};
2396             next;
2397
2398         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2399             $section = $1;
2400             if (defined($descr)) {
2401                 $descr =~ s/\s+$//;
2402                 $conf->{description} = $descr;
2403             }
2404             $descr = undef;
2405             $conf = $res->{snapshots}->{$section} = {};
2406             next;
2407         }
2408
2409         if ($line =~ m/^\#(.*)\s*$/) {
2410             $descr = '' if !defined($descr);
2411             $descr .= PVE::Tools::decode_text($1) . "\n";
2412             next;
2413         }
2414
2415         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2416             $descr = '' if !defined($descr);
2417             $descr .= PVE::Tools::decode_text($2);
2418         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2419             $conf->{snapstate} = $1;
2420         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2421             my $key = $1;
2422             my $value = $2;
2423             $conf->{$key} = $value;
2424         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2425             my $value = $1;
2426             if ($section eq 'pending') {
2427                 $conf->{delete} = $value; # we parse this later
2428             } else {
2429                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2430             }
2431         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2432             my $key = $1;
2433             my $value = $2;
2434             eval { $value = check_type($key, $value); };
2435             if ($@) {
2436                 warn "vm $vmid - unable to parse value of '$key' - $@";
2437             } else {
2438                 $key = 'ide2' if $key eq 'cdrom';
2439                 my $fmt = $confdesc->{$key}->{format};
2440                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2441                     my $v = parse_drive($key, $value);
2442                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2443                         $v->{file} = $volid;
2444                         $value = print_drive($vmid, $v);
2445                     } else {
2446                         warn "vm $vmid - unable to parse value of '$key'\n";
2447                         next;
2448                     }
2449                 }
2450
2451                 $conf->{$key} = $value;
2452             }
2453         }
2454     }
2455
2456     if (defined($descr)) {
2457         $descr =~ s/\s+$//;
2458         $conf->{description} = $descr;
2459     }
2460     delete $res->{snapstate}; # just to be sure
2461
2462     return $res;
2463 }
2464
2465 sub write_vm_config {
2466     my ($filename, $conf) = @_;
2467
2468     delete $conf->{snapstate}; # just to be sure
2469
2470     if ($conf->{cdrom}) {
2471         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2472         $conf->{ide2} = $conf->{cdrom};
2473         delete $conf->{cdrom};
2474     }
2475
2476     # we do not use 'smp' any longer
2477     if ($conf->{sockets}) {
2478         delete $conf->{smp};
2479     } elsif ($conf->{smp}) {
2480         $conf->{sockets} = $conf->{smp};
2481         delete $conf->{cores};
2482         delete $conf->{smp};
2483     }
2484
2485     my $used_volids = {};
2486
2487     my $cleanup_config = sub {
2488         my ($cref, $pending, $snapname) = @_;
2489
2490         foreach my $key (keys %$cref) {
2491             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2492                 $key eq 'snapstate' || $key eq 'pending';
2493             my $value = $cref->{$key};
2494             if ($key eq 'delete') {
2495                 die "propertry 'delete' is only allowed in [PENDING]\n"
2496                     if !$pending;
2497                 # fixme: check syntax?
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             die "unable to parse value of '$key' - $@" if $@;
2502
2503             $cref->{$key} = $value;
2504
2505             if (!$snapname && is_valid_drivename($key)) {
2506                 my $drive = parse_drive($key, $value);
2507                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2508             }
2509         }
2510     };
2511
2512     &$cleanup_config($conf);
2513
2514     &$cleanup_config($conf->{pending}, 1);
2515
2516     foreach my $snapname (keys %{$conf->{snapshots}}) {
2517         die "internal error" if $snapname eq 'pending';
2518         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2519     }
2520
2521     # remove 'unusedX' settings if we re-add a volume
2522     foreach my $key (keys %$conf) {
2523         my $value = $conf->{$key};
2524         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2525             delete $conf->{$key};
2526         }
2527     }
2528
2529     my $generate_raw_config = sub {
2530         my ($conf, $pending) = @_;
2531
2532         my $raw = '';
2533
2534         # add description as comment to top of file
2535         if (defined(my $descr = $conf->{description})) {
2536             if ($descr) {
2537                 foreach my $cl (split(/\n/, $descr)) {
2538                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2539                 }
2540             } else {
2541                 $raw .= "#\n" if $pending;
2542             }
2543         }
2544
2545         foreach my $key (sort keys %$conf) {
2546             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2547             $raw .= "$key: $conf->{$key}\n";
2548         }
2549         return $raw;
2550     };
2551
2552     my $raw = &$generate_raw_config($conf);
2553
2554     if (scalar(keys %{$conf->{pending}})){
2555         $raw .= "\n[PENDING]\n";
2556         $raw .= &$generate_raw_config($conf->{pending}, 1);
2557     }
2558
2559     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2560         $raw .= "\n[$snapname]\n";
2561         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2562     }
2563
2564     return $raw;
2565 }
2566
2567 sub load_defaults {
2568
2569     my $res = {};
2570
2571     # we use static defaults from our JSON schema configuration
2572     foreach my $key (keys %$confdesc) {
2573         if (defined(my $default = $confdesc->{$key}->{default})) {
2574             $res->{$key} = $default;
2575         }
2576     }
2577
2578     return $res;
2579 }
2580
2581 sub config_list {
2582     my $vmlist = PVE::Cluster::get_vmlist();
2583     my $res = {};
2584     return $res if !$vmlist || !$vmlist->{ids};
2585     my $ids = $vmlist->{ids};
2586
2587     foreach my $vmid (keys %$ids) {
2588         my $d = $ids->{$vmid};
2589         next if !$d->{node} || $d->{node} ne $nodename;
2590         next if !$d->{type} || $d->{type} ne 'qemu';
2591         $res->{$vmid}->{exists} = 1;
2592     }
2593     return $res;
2594 }
2595
2596 # test if VM uses local resources (to prevent migration)
2597 sub check_local_resources {
2598     my ($conf, $noerr) = @_;
2599
2600     my $loc_res = 0;
2601
2602     $loc_res = 1 if $conf->{hostusb}; # old syntax
2603     $loc_res = 1 if $conf->{hostpci}; # old syntax
2604
2605     foreach my $k (keys %$conf) {
2606         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2607         # sockets are safe: they will recreated be on the target side post-migrate
2608         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2609         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2610     }
2611
2612     die "VM uses local resources\n" if $loc_res && !$noerr;
2613
2614     return $loc_res;
2615 }
2616
2617 # check if used storages are available on all nodes (use by migrate)
2618 sub check_storage_availability {
2619     my ($storecfg, $conf, $node) = @_;
2620
2621     foreach_drive($conf, sub {
2622         my ($ds, $drive) = @_;
2623
2624         my $volid = $drive->{file};
2625         return if !$volid;
2626
2627         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2628         return if !$sid;
2629
2630         # check if storage is available on both nodes
2631         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2632         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2633    });
2634 }
2635
2636 # list nodes where all VM images are available (used by has_feature API)
2637 sub shared_nodes {
2638     my ($conf, $storecfg) = @_;
2639
2640     my $nodelist = PVE::Cluster::get_nodelist();
2641     my $nodehash = { map { $_ => 1 } @$nodelist };
2642     my $nodename = PVE::INotify::nodename();
2643
2644     foreach_drive($conf, sub {
2645         my ($ds, $drive) = @_;
2646
2647         my $volid = $drive->{file};
2648         return if !$volid;
2649
2650         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2651         if ($storeid) {
2652             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2653             if ($scfg->{disable}) {
2654                 $nodehash = {};
2655             } elsif (my $avail = $scfg->{nodes}) {
2656                 foreach my $node (keys %$nodehash) {
2657                     delete $nodehash->{$node} if !$avail->{$node};
2658                 }
2659             } elsif (!$scfg->{shared}) {
2660                 foreach my $node (keys %$nodehash) {
2661                     delete $nodehash->{$node} if $node ne $nodename
2662                 }
2663             }
2664         }
2665     });
2666
2667     return $nodehash
2668 }
2669
2670 sub check_cmdline {
2671     my ($pidfile, $pid) = @_;
2672
2673     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2674     if (defined($fh)) {
2675         my $line = <$fh>;
2676         $fh->close;
2677         return undef if !$line;
2678         my @param = split(/\0/, $line);
2679
2680         my $cmd = $param[0];
2681         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2682
2683         for (my $i = 0; $i < scalar (@param); $i++) {
2684             my $p = $param[$i];
2685             next if !$p;
2686             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2687                 my $p = $param[$i+1];
2688                 return 1 if $p && ($p eq $pidfile);
2689                 return undef;
2690             }
2691         }
2692     }
2693     return undef;
2694 }
2695
2696 sub check_running {
2697     my ($vmid, $nocheck, $node) = @_;
2698
2699     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2700
2701     die "unable to find configuration file for VM $vmid - no such machine\n"
2702         if !$nocheck && ! -f $filename;
2703
2704     my $pidfile = pidfile_name($vmid);
2705
2706     if (my $fd = IO::File->new("<$pidfile")) {
2707         my $st = stat($fd);
2708         my $line = <$fd>;
2709         close($fd);
2710
2711         my $mtime = $st->mtime;
2712         if ($mtime > time()) {
2713             warn "file '$filename' modified in future\n";
2714         }
2715
2716         if ($line =~ m/^(\d+)$/) {
2717             my $pid = $1;
2718             if (check_cmdline($pidfile, $pid)) {
2719                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2720                     return $pid;
2721                 }
2722             }
2723         }
2724     }
2725
2726     return undef;
2727 }
2728
2729 sub vzlist {
2730
2731     my $vzlist = config_list();
2732
2733     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2734
2735     while (defined(my $de = $fd->read)) {
2736         next if $de !~ m/^(\d+)\.pid$/;
2737         my $vmid = $1;
2738         next if !defined($vzlist->{$vmid});
2739         if (my $pid = check_running($vmid)) {
2740             $vzlist->{$vmid}->{pid} = $pid;
2741         }
2742     }
2743
2744     return $vzlist;
2745 }
2746
2747 sub disksize {
2748     my ($storecfg, $conf) = @_;
2749
2750     my $bootdisk = $conf->{bootdisk};
2751     return undef if !$bootdisk;
2752     return undef if !is_valid_drivename($bootdisk);
2753
2754     return undef if !$conf->{$bootdisk};
2755
2756     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2757     return undef if !defined($drive);
2758
2759     return undef if drive_is_cdrom($drive);
2760
2761     my $volid = $drive->{file};
2762     return undef if !$volid;
2763
2764     return $drive->{size};
2765 }
2766
2767 my $last_proc_pid_stat;
2768
2769 # get VM status information
2770 # This must be fast and should not block ($full == false)
2771 # We only query KVM using QMP if $full == true (this can be slow)
2772 sub vmstatus {
2773     my ($opt_vmid, $full) = @_;
2774
2775     my $res = {};
2776
2777     my $storecfg = PVE::Storage::config();
2778
2779     my $list = vzlist();
2780     my $defaults = load_defaults();
2781
2782     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2783
2784     my $cpucount = $cpuinfo->{cpus} || 1;
2785
2786     foreach my $vmid (keys %$list) {
2787         next if $opt_vmid && ($vmid ne $opt_vmid);
2788
2789         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2790         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2791
2792         my $d = {};
2793         $d->{pid} = $list->{$vmid}->{pid};
2794
2795         # fixme: better status?
2796         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2797
2798         my $size = disksize($storecfg, $conf);
2799         if (defined($size)) {
2800             $d->{disk} = 0; # no info available
2801             $d->{maxdisk} = $size;
2802         } else {
2803             $d->{disk} = 0;
2804             $d->{maxdisk} = 0;
2805         }
2806
2807         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2808             * ($conf->{cores} || $defaults->{cores});
2809         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2810         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2811
2812         $d->{name} = $conf->{name} || "VM $vmid";
2813         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2814             : $defaults->{memory}*(1024*1024);
2815
2816         if ($conf->{balloon}) {
2817             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2818             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2819                 : $defaults->{shares};
2820         }
2821
2822         $d->{uptime} = 0;
2823         $d->{cpu} = 0;
2824         $d->{mem} = 0;
2825
2826         $d->{netout} = 0;
2827         $d->{netin} = 0;
2828
2829         $d->{diskread} = 0;
2830         $d->{diskwrite} = 0;
2831
2832         $d->{template} = PVE::QemuConfig->is_template($conf);
2833
2834         $d->{serial} = 1 if conf_has_serial($conf);
2835
2836         $res->{$vmid} = $d;
2837     }
2838
2839     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2840     foreach my $dev (keys %$netdev) {
2841         next if $dev !~ m/^tap([1-9]\d*)i/;
2842         my $vmid = $1;
2843         my $d = $res->{$vmid};
2844         next if !$d;
2845
2846         $d->{netout} += $netdev->{$dev}->{receive};
2847         $d->{netin} += $netdev->{$dev}->{transmit};
2848
2849         if ($full) {
2850             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2851             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2852         }
2853
2854     }
2855
2856     my $ctime = gettimeofday;
2857
2858     foreach my $vmid (keys %$list) {
2859
2860         my $d = $res->{$vmid};
2861         my $pid = $d->{pid};
2862         next if !$pid;
2863
2864         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2865         next if !$pstat; # not running
2866
2867         my $used = $pstat->{utime} + $pstat->{stime};
2868
2869         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2870
2871         if ($pstat->{vsize}) {
2872             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2873         }
2874
2875         my $old = $last_proc_pid_stat->{$pid};
2876         if (!$old) {
2877             $last_proc_pid_stat->{$pid} = {
2878                 time => $ctime,
2879                 used => $used,
2880                 cpu => 0,
2881             };
2882             next;
2883         }
2884
2885         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2886
2887         if ($dtime > 1000) {
2888             my $dutime = $used -  $old->{used};
2889
2890             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => $d->{cpu},
2895             };
2896         } else {
2897             $d->{cpu} = $old->{cpu};
2898         }
2899     }
2900
2901     return $res if !$full;
2902
2903     my $qmpclient = PVE::QMPClient->new();
2904
2905     my $ballooncb = sub {
2906         my ($vmid, $resp) = @_;
2907
2908         my $info = $resp->{'return'};
2909         return if !$info->{max_mem};
2910
2911         my $d = $res->{$vmid};
2912
2913         # use memory assigned to VM
2914         $d->{maxmem} = $info->{max_mem};
2915         $d->{balloon} = $info->{actual};
2916
2917         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2918             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2919             $d->{freemem} = $info->{free_mem};
2920         }
2921
2922         $d->{ballooninfo} = $info;
2923     };
2924
2925     my $blockstatscb = sub {
2926         my ($vmid, $resp) = @_;
2927         my $data = $resp->{'return'} || [];
2928         my $totalrdbytes = 0;
2929         my $totalwrbytes = 0;
2930
2931         for my $blockstat (@$data) {
2932             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2933             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2934
2935             $blockstat->{device} =~ s/drive-//;
2936             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2937         }
2938         $res->{$vmid}->{diskread} = $totalrdbytes;
2939         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2940     };
2941
2942     my $statuscb = sub {
2943         my ($vmid, $resp) = @_;
2944
2945         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2946         # this fails if ballon driver is not loaded, so this must be
2947         # the last commnand (following command are aborted if this fails).
2948         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2949
2950         my $status = 'unknown';
2951         if (!defined($status = $resp->{'return'}->{status})) {
2952             warn "unable to get VM status\n";
2953             return;
2954         }
2955
2956         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2957     };
2958
2959     foreach my $vmid (keys %$list) {
2960         next if $opt_vmid && ($vmid ne $opt_vmid);
2961         next if !$res->{$vmid}->{pid}; # not running
2962         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2963     }
2964
2965     $qmpclient->queue_execute(undef, 2);
2966
2967     foreach my $vmid (keys %$list) {
2968         next if $opt_vmid && ($vmid ne $opt_vmid);
2969         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2970     }
2971
2972     return $res;
2973 }
2974
2975 sub foreach_drive {
2976     my ($conf, $func, @param) = @_;
2977
2978     foreach my $ds (valid_drive_names()) {
2979         next if !defined($conf->{$ds});
2980
2981         my $drive = parse_drive($ds, $conf->{$ds});
2982         next if !$drive;
2983
2984         &$func($ds, $drive, @param);
2985     }
2986 }
2987
2988 sub foreach_volid {
2989     my ($conf, $func, @param) = @_;
2990
2991     my $volhash = {};
2992
2993     my $test_volid = sub {
2994         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2995
2996         return if !$volid;
2997
2998         $volhash->{$volid}->{cdrom} //= 1;
2999         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3000
3001         $volhash->{$volid}->{replicate} //= 0;
3002         $volhash->{$volid}->{replicate} = 1 if $replicate;
3003
3004         $volhash->{$volid}->{shared} //= 0;
3005         $volhash->{$volid}->{shared} = 1 if $shared;
3006
3007         $volhash->{$volid}->{referenced_in_config} //= 0;
3008         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3009
3010         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3011             if defined($snapname);
3012     };
3013
3014     foreach_drive($conf, sub {
3015         my ($ds, $drive) = @_;
3016         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3017     });
3018
3019     foreach my $snapname (keys %{$conf->{snapshots}}) {
3020         my $snap = $conf->{snapshots}->{$snapname};
3021         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3022         foreach_drive($snap, sub {
3023             my ($ds, $drive) = @_;
3024             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3025         });
3026     }
3027
3028     foreach my $volid (keys %$volhash) {
3029         &$func($volid, $volhash->{$volid}, @param);
3030     }
3031 }
3032
3033 sub conf_has_serial {
3034     my ($conf) = @_;
3035
3036     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3037         if ($conf->{"serial$i"}) {
3038             return 1;
3039         }
3040     }
3041
3042     return 0;
3043 }
3044
3045 sub vga_conf_has_spice {
3046     my ($vga) = @_;
3047
3048     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3049
3050     return $1 || 1;
3051 }
3052
3053 sub config_to_command {
3054     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3055
3056     my $cmd = [];
3057     my $globalFlags = [];
3058     my $machineFlags = [];
3059     my $rtcFlags = [];
3060     my $cpuFlags = [];
3061     my $devices = [];
3062     my $pciaddr = '';
3063     my $bridges = {};
3064     my $kvmver = kvm_user_version();
3065     my $vernum = 0; # unknown
3066     my $ostype = $conf->{ostype};
3067     my $winversion = windows_version($ostype);
3068     my $kvm = $conf->{kvm} // 1;
3069
3070     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3071
3072     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3073         $vernum = $1*1000000+$2*1000;
3074     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3075         $vernum = $1*1000000+$2*1000+$3;
3076     }
3077
3078     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3079
3080     my $have_ovz = -f '/proc/vz/vestat';
3081
3082     my $q35 = machine_type_is_q35($conf);
3083     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3084     my $machine_type = $forcemachine || $conf->{machine};
3085     my $use_old_bios_files = undef;
3086     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3087
3088     my $cpuunits = defined($conf->{cpuunits}) ?
3089             $conf->{cpuunits} : $defaults->{cpuunits};
3090
3091     push @$cmd, '/usr/bin/kvm';
3092
3093     push @$cmd, '-id', $vmid;
3094
3095     my $vmname = $conf->{name} || "vm$vmid";
3096
3097     push @$cmd, '-name', $vmname;
3098
3099     my $use_virtio = 0;
3100
3101     my $qmpsocket = qmp_socket($vmid);
3102     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3103     push @$cmd, '-mon', "chardev=qmp,mode=control";
3104
3105
3106     push @$cmd, '-pidfile' , pidfile_name($vmid);
3107
3108     push @$cmd, '-daemonize';
3109
3110     if ($conf->{smbios1}) {
3111         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3112     }
3113
3114     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3115         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3116
3117         my $path;
3118         my $format;
3119         if (my $efidisk = $conf->{efidisk0}) {
3120             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3121             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3122             $format = $d->{format};
3123             if ($storeid) {
3124                 $path = PVE::Storage::path($storecfg, $d->{file});
3125                 if (!defined($format)) {
3126                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3127                     $format = qemu_img_format($scfg, $volname);
3128                 }
3129             } else {
3130                 $path = $d->{file};
3131                 die "efidisk format must be specified\n"
3132                     if !defined($format);
3133             }
3134         } else {
3135             warn "no efidisk configured! Using temporary efivars disk.\n";
3136             $path = "/tmp/$vmid-ovmf.fd";
3137             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3138             $format = 'raw';
3139         }
3140
3141         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3142         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3143     }
3144
3145
3146     # add usb controllers
3147     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3148     push @$devices, @usbcontrollers if @usbcontrollers;
3149     my $vga = $conf->{vga};
3150
3151     my $qxlnum = vga_conf_has_spice($vga);
3152     $vga = 'qxl' if $qxlnum;
3153
3154     if (!$vga) {
3155         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3156             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3157         } else {
3158             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3159         }
3160     }
3161
3162     # enable absolute mouse coordinates (needed by vnc)
3163     my $tablet;
3164     if (defined($conf->{tablet})) {
3165         $tablet = $conf->{tablet};
3166     } else {
3167         $tablet = $defaults->{tablet};
3168         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3169         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3170     }
3171
3172     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3173
3174     my $kvm_off = 0;
3175     my $gpu_passthrough;
3176
3177     # host pci devices
3178     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3179         my $d = parse_hostpci($conf->{"hostpci$i"});
3180         next if !$d;
3181
3182         my $pcie = $d->{pcie};
3183         if($pcie){
3184             die "q35 machine model is not enabled" if !$q35;
3185             $pciaddr = print_pcie_addr("hostpci$i");
3186         }else{
3187             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3188         }
3189
3190         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3191         my $romfile = $d->{romfile};
3192
3193         my $xvga = '';
3194         if ($d->{'x-vga'}) {
3195             $xvga = ',x-vga=on';
3196             $kvm_off = 1;
3197             $vga = 'none';
3198             $gpu_passthrough = 1;
3199
3200             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3201                 $xvga = "";
3202             }
3203         }
3204         my $pcidevices = $d->{pciid};
3205         my $multifunction = 1 if @$pcidevices > 1;
3206
3207         my $j=0;
3208         foreach my $pcidevice (@$pcidevices) {
3209
3210             my $id = "hostpci$i";
3211             $id .= ".$j" if $multifunction;
3212             my $addr = $pciaddr;
3213             $addr .= ".$j" if $multifunction;
3214             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3215
3216             if($j == 0){
3217                 $devicestr .= "$rombar$xvga";
3218                 $devicestr .= ",multifunction=on" if $multifunction;
3219                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3220             }
3221
3222             push @$devices, '-device', $devicestr;
3223             $j++;
3224         }
3225     }
3226
3227     # usb devices
3228     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3229     push @$devices, @usbdevices if @usbdevices;
3230     # serial devices
3231     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3232         if (my $path = $conf->{"serial$i"}) {
3233             if ($path eq 'socket') {
3234                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3235                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3236                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3237             } else {
3238                 die "no such serial device\n" if ! -c $path;
3239                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3240                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3241             }
3242         }
3243     }
3244
3245     # parallel devices
3246     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3247         if (my $path = $conf->{"parallel$i"}) {
3248             die "no such parallel device\n" if ! -c $path;
3249             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3250             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3251             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3252         }
3253     }
3254
3255
3256     my $sockets = 1;
3257     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3258     $sockets = $conf->{sockets} if  $conf->{sockets};
3259
3260     my $cores = $conf->{cores} || 1;
3261
3262     my $maxcpus = $sockets * $cores;
3263
3264     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3265
3266     my $allowed_vcpus = $cpuinfo->{cpus};
3267
3268     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3269         if ($allowed_vcpus < $maxcpus);
3270
3271     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3272
3273         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3274         for (my $i = 2; $i <= $vcpus; $i++)  {
3275             my $cpustr = print_cpu_device($conf,$i);
3276             push @$cmd, '-device', $cpustr;
3277         }
3278
3279     } else {
3280
3281         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3282     }
3283     push @$cmd, '-nodefaults';
3284
3285     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3286
3287     my $bootindex_hash = {};
3288     my $i = 1;
3289     foreach my $o (split(//, $bootorder)) {
3290         $bootindex_hash->{$o} = $i*100;
3291         $i++;
3292     }
3293
3294     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3301
3302     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3303         my $socket = vnc_socket($vmid);
3304         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3305     } else {
3306         push @$cmd, '-nographic';
3307     }
3308
3309     # time drift fix
3310     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3311
3312     my $useLocaltime = $conf->{localtime};
3313
3314     if ($winversion >= 5) { # windows
3315         $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317         # use time drift fix when acpi is enabled
3318         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319             $tdf = 1 if !defined($conf->{tdf});
3320         }
3321     }
3322
3323     if ($winversion >= 6) {
3324         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325         push @$cmd, '-no-hpet';
3326     }
3327
3328     push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330     if (!$kvm) {
3331         push @$machineFlags, 'accel=tcg';
3332     }
3333
3334     if ($machine_type) {
3335         push @$machineFlags, "type=${machine_type}";
3336     }
3337
3338     if ($conf->{startdate}) {
3339         push @$rtcFlags, "base=$conf->{startdate}";
3340     } elsif ($useLocaltime) {
3341         push @$rtcFlags, 'base=localtime';
3342     }
3343
3344     my $cpu = $kvm ? "kvm64" : "qemu64";
3345     if (my $cputype = $conf->{cpu}) {
3346         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3347             or die "Cannot parse cpu description: $cputype\n";
3348         $cpu = $cpuconf->{cputype};
3349         $kvm_off = 1 if $cpuconf->{hidden};
3350
3351         if (defined(my $flags = $cpuconf->{flags})) {
3352             push @$cpuFlags, split(";", $flags);
3353         }
3354     }
3355
3356     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3357
3358     push @$cpuFlags , '-x2apic'
3359         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3360
3361     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3362
3363     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3364
3365     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3366
3367         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3368         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3369     }
3370
3371     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3372
3373     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3374
3375     push @$cpuFlags, 'kvm=off' if $kvm_off;
3376
3377     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3378         die "internal error"; # should not happen
3379
3380     push @$cpuFlags, "vendor=${cpu_vendor}"
3381         if $cpu_vendor ne 'default';
3382
3383     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3384
3385     push @$cmd, '-cpu', $cpu;
3386
3387     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3388
3389     push @$cmd, '-S' if $conf->{freeze};
3390
3391     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3392
3393     # enable sound
3394     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3395     #push @$cmd, '-soundhw', 'es1370';
3396     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3397
3398     if($conf->{agent}) {
3399         my $qgasocket = qmp_socket($vmid, 1);
3400         my $pciaddr = print_pci_addr("qga0", $bridges);
3401         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3402         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3403         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3404     }
3405
3406     my $spice_port;
3407
3408     if ($qxlnum) {
3409         if ($qxlnum > 1) {
3410             if ($winversion){
3411                 for(my $i = 1; $i < $qxlnum; $i++){
3412                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3413                     push @$devices, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3414                 }
3415             } else {
3416                 # assume other OS works like Linux
3417                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3418                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3419             }
3420         }
3421
3422         my $pciaddr = print_pci_addr("spice", $bridges);
3423
3424         my $nodename = PVE::INotify::nodename();
3425         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3426         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3427         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3428         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3429         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3430
3431         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3432
3433         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3434         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3435         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3436     }
3437
3438     # enable balloon by default, unless explicitly disabled
3439     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3440         $pciaddr = print_pci_addr("balloon0", $bridges);
3441         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3442     }
3443
3444     if ($conf->{watchdog}) {
3445         my $wdopts = parse_watchdog($conf->{watchdog});
3446         $pciaddr = print_pci_addr("watchdog", $bridges);
3447         my $watchdog = $wdopts->{model} || 'i6300esb';
3448         push @$devices, '-device', "$watchdog$pciaddr";
3449         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3450     }
3451
3452     my $vollist = [];
3453     my $scsicontroller = {};
3454     my $ahcicontroller = {};
3455     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3456
3457     # Add iscsi initiator name if available
3458     if (my $initiator = get_initiator_name()) {
3459         push @$devices, '-iscsi', "initiator-name=$initiator";
3460     }
3461
3462     foreach_drive($conf, sub {
3463         my ($ds, $drive) = @_;
3464
3465         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3466             push @$vollist, $drive->{file};
3467         }
3468
3469         # ignore efidisk here, already added in bios/fw handling code above
3470         return if $drive->{interface} eq 'efidisk';
3471
3472         $use_virtio = 1 if $ds =~ m/^virtio/;
3473
3474         if (drive_is_cdrom ($drive)) {
3475             if ($bootindex_hash->{d}) {
3476                 $drive->{bootindex} = $bootindex_hash->{d};
3477                 $bootindex_hash->{d} += 1;
3478             }
3479         } else {
3480             if ($bootindex_hash->{c}) {
3481                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3482                 $bootindex_hash->{c} += 1;
3483             }
3484         }
3485
3486         if($drive->{interface} eq 'virtio'){
3487            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3488         }
3489
3490         if ($drive->{interface} eq 'scsi') {
3491
3492             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3493
3494             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3495             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3496
3497             my $iothread = '';
3498             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3499                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3500                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3501             } elsif ($drive->{iothread}) {
3502                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3503             }
3504
3505             my $queues = '';
3506             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3507                 $queues = ",num_queues=$drive->{queues}";
3508             }
3509
3510             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3511             $scsicontroller->{$controller}=1;
3512         }
3513
3514         if ($drive->{interface} eq 'sata') {
3515            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3516            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3517            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3518            $ahcicontroller->{$controller}=1;
3519         }
3520
3521         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3522         push @$devices, '-drive',$drive_cmd;
3523         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3524     });
3525
3526     for (my $i = 0; $i < $MAX_NETS; $i++) {
3527          next if !$conf->{"net$i"};
3528          my $d = parse_net($conf->{"net$i"});
3529          next if !$d;
3530
3531          $use_virtio = 1 if $d->{model} eq 'virtio';
3532
3533          if ($bootindex_hash->{n}) {
3534             $d->{bootindex} = $bootindex_hash->{n};
3535             $bootindex_hash->{n} += 1;
3536          }
3537
3538          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3539          push @$devices, '-netdev', $netdevfull;
3540
3541          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3542          push @$devices, '-device', $netdevicefull;
3543     }
3544
3545     if (!$q35) {
3546         # add pci bridges
3547         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3548            $bridges->{1} = 1;
3549            $bridges->{2} = 1;
3550         }
3551
3552         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3553
3554         while (my ($k, $v) = each %$bridges) {
3555             $pciaddr = print_pci_addr("pci.$k");
3556             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3557         }
3558     }
3559
3560     # add custom args
3561     if ($conf->{args}) {
3562         my $aa = PVE::Tools::split_args($conf->{args});
3563         push @$cmd, @$aa;
3564     }
3565
3566     push @$cmd, @$devices;
3567     push @$cmd, '-rtc', join(',', @$rtcFlags)
3568         if scalar(@$rtcFlags);
3569     push @$cmd, '-machine', join(',', @$machineFlags)
3570         if scalar(@$machineFlags);
3571     push @$cmd, '-global', join(',', @$globalFlags)
3572         if scalar(@$globalFlags);
3573
3574     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3575 }
3576
3577 sub vnc_socket {
3578     my ($vmid) = @_;
3579     return "${var_run_tmpdir}/$vmid.vnc";
3580 }
3581
3582 sub spice_port {
3583     my ($vmid) = @_;
3584
3585     my $res = vm_mon_cmd($vmid, 'query-spice');
3586
3587     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3588 }
3589
3590 sub qmp_socket {
3591     my ($vmid, $qga) = @_;
3592     my $sockettype = $qga ? 'qga' : 'qmp';
3593     return "${var_run_tmpdir}/$vmid.$sockettype";
3594 }
3595
3596 sub pidfile_name {
3597     my ($vmid) = @_;
3598     return "${var_run_tmpdir}/$vmid.pid";
3599 }
3600
3601 sub vm_devices_list {
3602     my ($vmid) = @_;
3603
3604     my $res = vm_mon_cmd($vmid, 'query-pci');
3605     my $devices_to_check = [];
3606     my $devices = {};
3607     foreach my $pcibus (@$res) {
3608         push @$devices_to_check, @{$pcibus->{devices}},
3609     }
3610
3611     while (@$devices_to_check) {
3612         my $to_check = [];
3613         for my $d (@$devices_to_check) {
3614             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3615             next if !$d->{'pci_bridge'};
3616
3617             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3618             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3619         }
3620         $devices_to_check = $to_check;
3621     }
3622
3623     my $resblock = vm_mon_cmd($vmid, 'query-block');
3624     foreach my $block (@$resblock) {
3625         if($block->{device} =~ m/^drive-(\S+)/){
3626                 $devices->{$1} = 1;
3627         }
3628     }
3629
3630     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3631     foreach my $mice (@$resmice) {
3632         if ($mice->{name} eq 'QEMU HID Tablet') {
3633             $devices->{tablet} = 1;
3634             last;
3635         }
3636     }
3637
3638     # for usb devices there is no query-usb
3639     # but we can iterate over the entries in
3640     # qom-list path=/machine/peripheral
3641     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3642     foreach my $per (@$resperipheral) {
3643         if ($per->{name} =~ m/^usb\d+$/) {
3644             $devices->{$per->{name}} = 1;
3645         }
3646     }
3647
3648     return $devices;
3649 }
3650
3651 sub vm_deviceplug {
3652     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3653
3654     my $q35 = machine_type_is_q35($conf);
3655
3656     my $devices_list = vm_devices_list($vmid);
3657     return 1 if defined($devices_list->{$deviceid});
3658
3659     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3660
3661     if ($deviceid eq 'tablet') {
3662
3663         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3664
3665     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3666
3667         die "usb hotplug currently not reliable\n";
3668         # since we can't reliably hot unplug all added usb devices
3669         # and usb passthrough disables live migration
3670         # we disable usb hotplugging for now
3671         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3672
3673     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3674
3675         qemu_iothread_add($vmid, $deviceid, $device);
3676
3677         qemu_driveadd($storecfg, $vmid, $device);
3678         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3679
3680         qemu_deviceadd($vmid, $devicefull);
3681         eval { qemu_deviceaddverify($vmid, $deviceid); };
3682         if (my $err = $@) {
3683             eval { qemu_drivedel($vmid, $deviceid); };
3684             warn $@ if $@;
3685             die $err;
3686         }
3687
3688     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3689
3690
3691         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3692         my $pciaddr = print_pci_addr($deviceid);
3693         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3694
3695         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3696
3697         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3698             qemu_iothread_add($vmid, $deviceid, $device);
3699             $devicefull .= ",iothread=iothread-$deviceid";
3700         }
3701
3702         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3703             $devicefull .= ",num_queues=$device->{queues}";
3704         }
3705
3706         qemu_deviceadd($vmid, $devicefull);
3707         qemu_deviceaddverify($vmid, $deviceid);
3708
3709     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3710
3711         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3712         qemu_driveadd($storecfg, $vmid, $device);
3713
3714         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3715         eval { qemu_deviceadd($vmid, $devicefull); };
3716         if (my $err = $@) {
3717             eval { qemu_drivedel($vmid, $deviceid); };
3718             warn $@ if $@;
3719             die $err;
3720         }
3721
3722     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3723
3724         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3725
3726         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3727         my $use_old_bios_files = undef;
3728         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3729
3730         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3731         qemu_deviceadd($vmid, $netdevicefull);
3732         eval { qemu_deviceaddverify($vmid, $deviceid); };
3733         if (my $err = $@) {
3734             eval { qemu_netdevdel($vmid, $deviceid); };
3735             warn $@ if $@;
3736             die $err;
3737         }
3738
3739     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3740
3741         my $bridgeid = $2;
3742         my $pciaddr = print_pci_addr($deviceid);
3743         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3744
3745         qemu_deviceadd($vmid, $devicefull);
3746         qemu_deviceaddverify($vmid, $deviceid);
3747
3748     } else {
3749         die "can't hotplug device '$deviceid'\n";
3750     }
3751
3752     return 1;
3753 }
3754
3755 # fixme: this should raise exceptions on error!
3756 sub vm_deviceunplug {
3757     my ($vmid, $conf, $deviceid) = @_;
3758
3759     my $devices_list = vm_devices_list($vmid);
3760     return 1 if !defined($devices_list->{$deviceid});
3761
3762     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3763
3764     if ($deviceid eq 'tablet') {
3765
3766         qemu_devicedel($vmid, $deviceid);
3767
3768     } elsif ($deviceid =~ m/^usb\d+$/) {
3769
3770         die "usb hotplug currently not reliable\n";
3771         # when unplugging usb devices this way,
3772         # there may be remaining usb controllers/hubs
3773         # so we disable it for now
3774         qemu_devicedel($vmid, $deviceid);
3775         qemu_devicedelverify($vmid, $deviceid);
3776
3777     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3778
3779         qemu_devicedel($vmid, $deviceid);
3780         qemu_devicedelverify($vmid, $deviceid);
3781         qemu_drivedel($vmid, $deviceid);
3782         qemu_iothread_del($conf, $vmid, $deviceid);
3783
3784     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3785
3786         qemu_devicedel($vmid, $deviceid);
3787         qemu_devicedelverify($vmid, $deviceid);
3788         qemu_iothread_del($conf, $vmid, $deviceid);
3789
3790     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3791
3792         qemu_devicedel($vmid, $deviceid);
3793         qemu_drivedel($vmid, $deviceid);
3794         qemu_deletescsihw($conf, $vmid, $deviceid);
3795
3796     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3797
3798         qemu_devicedel($vmid, $deviceid);
3799         qemu_devicedelverify($vmid, $deviceid);
3800         qemu_netdevdel($vmid, $deviceid);
3801
3802     } else {
3803         die "can't unplug device '$deviceid'\n";
3804     }
3805
3806     return 1;
3807 }
3808
3809 sub qemu_deviceadd {
3810     my ($vmid, $devicefull) = @_;
3811
3812     $devicefull = "driver=".$devicefull;
3813     my %options =  split(/[=,]/, $devicefull);
3814
3815     vm_mon_cmd($vmid, "device_add" , %options);
3816 }
3817
3818 sub qemu_devicedel {
3819     my ($vmid, $deviceid) = @_;
3820
3821     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3822 }
3823
3824 sub qemu_iothread_add {
3825     my($vmid, $deviceid, $device) = @_;
3826
3827     if ($device->{iothread}) {
3828         my $iothreads = vm_iothreads_list($vmid);
3829         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3830     }
3831 }
3832
3833 sub qemu_iothread_del {
3834     my($conf, $vmid, $deviceid) = @_;
3835
3836     my $device = parse_drive($deviceid, $conf->{$deviceid});
3837     if ($device->{iothread}) {
3838         my $iothreads = vm_iothreads_list($vmid);
3839         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3840     }
3841 }
3842
3843 sub qemu_objectadd {
3844     my($vmid, $objectid, $qomtype) = @_;
3845
3846     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3847
3848     return 1;
3849 }
3850
3851 sub qemu_objectdel {
3852     my($vmid, $objectid) = @_;
3853
3854     vm_mon_cmd($vmid, "object-del", id => $objectid);
3855
3856     return 1;
3857 }
3858
3859 sub qemu_driveadd {
3860     my ($storecfg, $vmid, $device) = @_;
3861
3862     my $drive = print_drive_full($storecfg, $vmid, $device);
3863     $drive =~ s/\\/\\\\/g;
3864     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3865
3866     # If the command succeeds qemu prints: "OK"
3867     return 1 if $ret =~ m/OK/s;
3868
3869     die "adding drive failed: $ret\n";
3870 }
3871
3872 sub qemu_drivedel {
3873     my($vmid, $deviceid) = @_;
3874
3875     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3876     $ret =~ s/^\s+//;
3877
3878     return 1 if $ret eq "";
3879
3880     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3881     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3882
3883     die "deleting drive $deviceid failed : $ret\n";
3884 }
3885
3886 sub qemu_deviceaddverify {
3887     my ($vmid, $deviceid) = @_;
3888
3889     for (my $i = 0; $i <= 5; $i++) {
3890          my $devices_list = vm_devices_list($vmid);
3891          return 1 if defined($devices_list->{$deviceid});
3892          sleep 1;
3893     }
3894
3895     die "error on hotplug device '$deviceid'\n";
3896 }
3897
3898
3899 sub qemu_devicedelverify {
3900     my ($vmid, $deviceid) = @_;
3901
3902     # need to verify that the device is correctly removed as device_del
3903     # is async and empty return is not reliable
3904
3905     for (my $i = 0; $i <= 5; $i++) {
3906          my $devices_list = vm_devices_list($vmid);
3907          return 1 if !defined($devices_list->{$deviceid});
3908          sleep 1;
3909     }
3910
3911     die "error on hot-unplugging device '$deviceid'\n";
3912 }
3913
3914 sub qemu_findorcreatescsihw {
3915     my ($storecfg, $conf, $vmid, $device) = @_;
3916
3917     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3918
3919     my $scsihwid="$controller_prefix$controller";
3920     my $devices_list = vm_devices_list($vmid);
3921
3922     if(!defined($devices_list->{$scsihwid})) {
3923         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3924     }
3925
3926     return 1;
3927 }
3928
3929 sub qemu_deletescsihw {
3930     my ($conf, $vmid, $opt) = @_;
3931
3932     my $device = parse_drive($opt, $conf->{$opt});
3933
3934     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3935         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3936         return 1;
3937     }
3938
3939     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3940
3941     my $devices_list = vm_devices_list($vmid);
3942     foreach my $opt (keys %{$devices_list}) {
3943         if (PVE::QemuServer::is_valid_drivename($opt)) {
3944             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3945             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3946                 return 1;
3947             }
3948         }
3949     }
3950
3951     my $scsihwid="scsihw$controller";
3952
3953     vm_deviceunplug($vmid, $conf, $scsihwid);
3954
3955     return 1;
3956 }
3957
3958 sub qemu_add_pci_bridge {
3959     my ($storecfg, $conf, $vmid, $device) = @_;
3960
3961     my $bridges = {};
3962
3963     my $bridgeid;
3964
3965     print_pci_addr($device, $bridges);
3966
3967     while (my ($k, $v) = each %$bridges) {
3968         $bridgeid = $k;
3969     }
3970     return 1 if !defined($bridgeid) || $bridgeid < 1;
3971
3972     my $bridge = "pci.$bridgeid";
3973     my $devices_list = vm_devices_list($vmid);
3974
3975     if (!defined($devices_list->{$bridge})) {
3976         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3977     }
3978
3979     return 1;
3980 }
3981
3982 sub qemu_set_link_status {
3983     my ($vmid, $device, $up) = @_;
3984
3985     vm_mon_cmd($vmid, "set_link", name => $device,
3986                up => $up ? JSON::true : JSON::false);
3987 }
3988
3989 sub qemu_netdevadd {
3990     my ($vmid, $conf, $device, $deviceid) = @_;
3991
3992     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3993     my %options =  split(/[=,]/, $netdev);
3994
3995     vm_mon_cmd($vmid, "netdev_add",  %options);
3996     return 1;
3997 }
3998
3999 sub qemu_netdevdel {
4000     my ($vmid, $deviceid) = @_;
4001
4002     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4003 }
4004
4005 sub qemu_usb_hotplug {
4006     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4007
4008     return if !$device;
4009
4010     # remove the old one first
4011     vm_deviceunplug($vmid, $conf, $deviceid);
4012
4013     # check if xhci controller is necessary and available
4014     if ($device->{usb3}) {
4015
4016         my $devicelist = vm_devices_list($vmid);
4017
4018         if (!$devicelist->{xhci}) {
4019             my $pciaddr = print_pci_addr("xhci");
4020             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4021         }
4022     }
4023     my $d = parse_usb_device($device->{host});
4024     $d->{usb3} = $device->{usb3};
4025
4026     # add the new one
4027     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4028 }
4029
4030 sub qemu_cpu_hotplug {
4031     my ($vmid, $conf, $vcpus) = @_;
4032
4033     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4034
4035     my $sockets = 1;
4036     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4037     $sockets = $conf->{sockets} if  $conf->{sockets};
4038     my $cores = $conf->{cores} || 1;
4039     my $maxcpus = $sockets * $cores;
4040
4041     $vcpus = $maxcpus if !$vcpus;
4042
4043     die "you can't add more vcpus than maxcpus\n"
4044         if $vcpus > $maxcpus;
4045
4046     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4047
4048     if ($vcpus < $currentvcpus) {
4049
4050         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4051
4052             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4053                 qemu_devicedel($vmid, "cpu$i");
4054                 my $retry = 0;
4055                 my $currentrunningvcpus = undef;
4056                 while (1) {
4057                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4058                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4059                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4060                     $retry++;
4061                     sleep 1;
4062                 }
4063                 #update conf after each succesfull cpu unplug
4064                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4065                 PVE::QemuConfig->write_config($vmid, $conf);
4066             }
4067         } else {
4068             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4069         }
4070
4071         return;
4072     }
4073
4074     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4075     die "vcpus in running vm does not match its configuration\n"
4076         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4077
4078     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4079
4080         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4081             my $cpustr = print_cpu_device($conf, $i);
4082             qemu_deviceadd($vmid, $cpustr);
4083
4084             my $retry = 0;
4085             my $currentrunningvcpus = undef;
4086             while (1) {
4087                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4088                 last if scalar(@{$currentrunningvcpus}) == $i;
4089                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4090                 sleep 1;
4091                 $retry++;
4092             }
4093             #update conf after each succesfull cpu hotplug
4094             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4095             PVE::QemuConfig->write_config($vmid, $conf);
4096         }
4097     } else {
4098
4099         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4100             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4101         }
4102     }
4103 }
4104
4105 sub qemu_block_set_io_throttle {
4106     my ($vmid, $deviceid,
4107         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4108         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4109         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4110         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4111
4112     return if !check_running($vmid) ;
4113
4114     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4115         bps => int($bps),
4116         bps_rd => int($bps_rd),
4117         bps_wr => int($bps_wr),
4118         iops => int($iops),
4119         iops_rd => int($iops_rd),
4120         iops_wr => int($iops_wr),
4121         bps_max => int($bps_max),
4122         bps_rd_max => int($bps_rd_max),
4123         bps_wr_max => int($bps_wr_max),
4124         iops_max => int($iops_max),
4125         iops_rd_max => int($iops_rd_max),
4126         iops_wr_max => int($iops_wr_max),
4127         bps_max_length => int($bps_max_length),
4128         bps_rd_max_length => int($bps_rd_max_length),
4129         bps_wr_max_length => int($bps_wr_max_length),
4130         iops_max_length => int($iops_max_length),
4131         iops_rd_max_length => int($iops_rd_max_length),
4132         iops_wr_max_length => int($iops_wr_max_length),
4133     );
4134
4135 }
4136
4137 # old code, only used to shutdown old VM after update
4138 sub __read_avail {
4139     my ($fh, $timeout) = @_;
4140
4141     my $sel = new IO::Select;
4142     $sel->add($fh);
4143
4144     my $res = '';
4145     my $buf;
4146
4147     my @ready;
4148     while (scalar (@ready = $sel->can_read($timeout))) {
4149         my $count;
4150         if ($count = $fh->sysread($buf, 8192)) {
4151             if ($buf =~ /^(.*)\(qemu\) $/s) {
4152                 $res .= $1;
4153                 last;
4154             } else {
4155                 $res .= $buf;
4156             }
4157         } else {
4158             if (!defined($count)) {
4159                 die "$!\n";
4160             }
4161             last;
4162         }
4163     }
4164
4165     die "monitor read timeout\n" if !scalar(@ready);
4166
4167     return $res;
4168 }
4169
4170 sub qemu_block_resize {
4171     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4172
4173     my $running = check_running($vmid);
4174
4175     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4176
4177     return if !$running;
4178
4179     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4180
4181 }
4182
4183 sub qemu_volume_snapshot {
4184     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4185
4186     my $running = check_running($vmid);
4187
4188     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4189         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4190     } else {
4191         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4192     }
4193 }
4194
4195 sub qemu_volume_snapshot_delete {
4196     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4197
4198     my $running = check_running($vmid);
4199
4200     if($running) {
4201
4202         $running = undef;
4203         my $conf = PVE::QemuConfig->load_config($vmid);
4204         foreach_drive($conf, sub {
4205             my ($ds, $drive) = @_;
4206             $running = 1 if $drive->{file} eq $volid;
4207         });
4208     }
4209
4210     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4211         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4212     } else {
4213         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4214     }
4215 }
4216
4217 sub set_migration_caps {
4218     my ($vmid) = @_;
4219
4220     my $cap_ref = [];
4221
4222     my $enabled_cap = {
4223         "auto-converge" => 1,
4224         "xbzrle" => 1,
4225         "x-rdma-pin-all" => 0,
4226         "zero-blocks" => 0,
4227         "compress" => 0
4228     };
4229
4230     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4231
4232     for my $supported_capability (@$supported_capabilities) {
4233         push @$cap_ref, {
4234             capability => $supported_capability->{capability},
4235             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4236         };
4237     }
4238
4239     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4240 }
4241
4242 my $fast_plug_option = {
4243     'lock' => 1,
4244     'name' => 1,
4245     'onboot' => 1,
4246     'shares' => 1,
4247     'startup' => 1,
4248     'description' => 1,
4249     'protection' => 1,
4250     'vmstatestorage' => 1,
4251 };
4252
4253 # hotplug changes in [PENDING]
4254 # $selection hash can be used to only apply specified options, for
4255 # example: { cores => 1 } (only apply changed 'cores')
4256 # $errors ref is used to return error messages
4257 sub vmconfig_hotplug_pending {
4258     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4259
4260     my $defaults = load_defaults();
4261
4262     # commit values which do not have any impact on running VM first
4263     # Note: those option cannot raise errors, we we do not care about
4264     # $selection and always apply them.
4265
4266     my $add_error = sub {
4267         my ($opt, $msg) = @_;
4268         $errors->{$opt} = "hotplug problem - $msg";
4269     };
4270
4271     my $changes = 0;
4272     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4273         if ($fast_plug_option->{$opt}) {
4274             $conf->{$opt} = $conf->{pending}->{$opt};
4275             delete $conf->{pending}->{$opt};
4276             $changes = 1;
4277         }
4278     }
4279
4280     if ($changes) {
4281         PVE::QemuConfig->write_config($vmid, $conf);
4282         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4283     }
4284
4285     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4286
4287     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4288     while (my ($opt, $force) = each %$pending_delete_hash) {
4289         next if $selection && !$selection->{$opt};
4290         eval {
4291             if ($opt eq 'hotplug') {
4292                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4293             } elsif ($opt eq 'tablet') {
4294                 die "skip\n" if !$hotplug_features->{usb};
4295                 if ($defaults->{tablet}) {
4296                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4297                 } else {
4298                     vm_deviceunplug($vmid, $conf, $opt);
4299                 }
4300             } elsif ($opt =~ m/^usb\d+/) {
4301                 die "skip\n";
4302                 # since we cannot reliably hot unplug usb devices
4303                 # we are disabling it
4304                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4305                 vm_deviceunplug($vmid, $conf, $opt);
4306             } elsif ($opt eq 'vcpus') {
4307                 die "skip\n" if !$hotplug_features->{cpu};
4308                 qemu_cpu_hotplug($vmid, $conf, undef);
4309             } elsif ($opt eq 'balloon') {
4310                 # enable balloon device is not hotpluggable
4311                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4312                 # here we reset the ballooning value to memory
4313                 my $balloon = $conf->{memory} || $defaults->{memory};
4314                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4315             } elsif ($fast_plug_option->{$opt}) {
4316                 # do nothing
4317             } elsif ($opt =~ m/^net(\d+)$/) {
4318                 die "skip\n" if !$hotplug_features->{network};
4319                 vm_deviceunplug($vmid, $conf, $opt);
4320             } elsif (is_valid_drivename($opt)) {
4321                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4322                 vm_deviceunplug($vmid, $conf, $opt);
4323                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4324             } elsif ($opt =~ m/^memory$/) {
4325                 die "skip\n" if !$hotplug_features->{memory};
4326                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4327             } elsif ($opt eq 'cpuunits') {
4328                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4329             } elsif ($opt eq 'cpulimit') {
4330                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4331             } else {
4332                 die "skip\n";
4333             }
4334         };
4335         if (my $err = $@) {
4336             &$add_error($opt, $err) if $err ne "skip\n";
4337         } else {
4338             # save new config if hotplug was successful
4339             delete $conf->{$opt};
4340             vmconfig_undelete_pending_option($conf, $opt);
4341             PVE::QemuConfig->write_config($vmid, $conf);
4342             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4343         }
4344     }
4345
4346     my $apply_pending_cloudinit;
4347     $apply_pending_cloudinit = sub {
4348         my ($key, $value) = @_;
4349         $apply_pending_cloudinit = sub {}; # once is enough
4350
4351         my @cloudinit_opts = keys %$confdesc_cloudinit;
4352         foreach my $opt (keys %{$conf->{pending}}) {
4353             next if !grep { $_ eq $opt } @cloudinit_opts;
4354             $conf->{$opt} = delete $conf->{pending}->{$opt};
4355         }
4356
4357         my $new_conf = { %$conf };
4358         $new_conf->{$key} = $value;
4359         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4360     };
4361
4362     foreach my $opt (keys %{$conf->{pending}}) {
4363         next if $selection && !$selection->{$opt};
4364         my $value = $conf->{pending}->{$opt};
4365         eval {
4366             if ($opt eq 'hotplug') {
4367                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4368             } elsif ($opt eq 'tablet') {
4369                 die "skip\n" if !$hotplug_features->{usb};
4370                 if ($value == 1) {
4371                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4372                 } elsif ($value == 0) {
4373                     vm_deviceunplug($vmid, $conf, $opt);
4374                 }
4375             } elsif ($opt =~ m/^usb\d+$/) {
4376                 die "skip\n";
4377                 # since we cannot reliably hot unplug usb devices
4378                 # we are disabling it
4379                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4380                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4381                 die "skip\n" if !$d;
4382                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4383             } elsif ($opt eq 'vcpus') {
4384                 die "skip\n" if !$hotplug_features->{cpu};
4385                 qemu_cpu_hotplug($vmid, $conf, $value);
4386             } elsif ($opt eq 'balloon') {
4387                 # enable/disable balloning device is not hotpluggable
4388                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4389                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4390                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4391
4392                 # allow manual ballooning if shares is set to zero
4393                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4394                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4395                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4396                 }
4397             } elsif ($opt =~ m/^net(\d+)$/) {
4398                 # some changes can be done without hotplug
4399                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4400                                     $vmid, $opt, $value);
4401             } elsif (is_valid_drivename($opt)) {
4402                 # some changes can be done without hotplug
4403                 my $drive = parse_drive($opt, $value);
4404                 if (drive_is_cloudinit($drive)) {
4405                     &$apply_pending_cloudinit($opt, $value);
4406                 }
4407                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4408                                      $vmid, $opt, $value, 1);
4409             } elsif ($opt =~ m/^memory$/) { #dimms
4410                 die "skip\n" if !$hotplug_features->{memory};
4411                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4412             } elsif ($opt eq 'cpuunits') {
4413                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4414             } elsif ($opt eq 'cpulimit') {
4415                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4416                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4417             } else {
4418                 die "skip\n";  # skip non-hot-pluggable options
4419             }
4420         };
4421         if (my $err = $@) {
4422             &$add_error($opt, $err) if $err ne "skip\n";
4423         } else {
4424             # save new config if hotplug was successful
4425             $conf->{$opt} = $value;
4426             delete $conf->{pending}->{$opt};
4427             PVE::QemuConfig->write_config($vmid, $conf);
4428             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4429         }
4430     }
4431 }
4432
4433 sub try_deallocate_drive {
4434     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4435
4436     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4437         my $volid = $drive->{file};
4438         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4439             my $sid = PVE::Storage::parse_volume_id($volid);
4440             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4441
4442             # check if the disk is really unused
4443             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4444                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4445             PVE::Storage::vdisk_free($storecfg, $volid);
4446             return 1;
4447         } else {
4448             # If vm is not owner of this disk remove from config
4449             return 1;
4450         }
4451     }
4452
4453     return undef;
4454 }
4455
4456 sub vmconfig_delete_or_detach_drive {
4457     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4458
4459     my $drive = parse_drive($opt, $conf->{$opt});
4460
4461     my $rpcenv = PVE::RPCEnvironment::get();
4462     my $authuser = $rpcenv->get_user();
4463
4464     if ($force) {
4465         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4466         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4467     } else {
4468         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4469     }
4470 }
4471
4472 sub vmconfig_apply_pending {
4473     my ($vmid, $conf, $storecfg) = @_;
4474
4475     # cold plug
4476
4477     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4478     while (my ($opt, $force) = each %$pending_delete_hash) {
4479         die "internal error" if $opt =~ m/^unused/;
4480         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4481         if (!defined($conf->{$opt})) {
4482             vmconfig_undelete_pending_option($conf, $opt);
4483             PVE::QemuConfig->write_config($vmid, $conf);
4484         } elsif (is_valid_drivename($opt)) {
4485             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4486             vmconfig_undelete_pending_option($conf, $opt);
4487             delete $conf->{$opt};
4488             PVE::QemuConfig->write_config($vmid, $conf);
4489         } else {
4490             vmconfig_undelete_pending_option($conf, $opt);
4491             delete $conf->{$opt};
4492             PVE::QemuConfig->write_config($vmid, $conf);
4493         }
4494     }
4495
4496     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4497
4498     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4499         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4500
4501         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4502             # skip if nothing changed
4503         } elsif (is_valid_drivename($opt)) {
4504             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4505                 if defined($conf->{$opt});
4506             $conf->{$opt} = $conf->{pending}->{$opt};
4507         } else {
4508             $conf->{$opt} = $conf->{pending}->{$opt};
4509         }
4510
4511         delete $conf->{pending}->{$opt};
4512         PVE::QemuConfig->write_config($vmid, $conf);
4513     }
4514 }
4515
4516 my $safe_num_ne = sub {
4517     my ($a, $b) = @_;
4518
4519     return 0 if !defined($a) && !defined($b);
4520     return 1 if !defined($a);
4521     return 1 if !defined($b);
4522
4523     return $a != $b;
4524 };
4525
4526 my $safe_string_ne = sub {
4527     my ($a, $b) = @_;
4528
4529     return 0 if !defined($a) && !defined($b);
4530     return 1 if !defined($a);
4531     return 1 if !defined($b);
4532
4533     return $a ne $b;
4534 };
4535
4536 sub vmconfig_update_net {
4537     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4538
4539     my $newnet = parse_net($value);
4540
4541     if ($conf->{$opt}) {
4542         my $oldnet = parse_net($conf->{$opt});
4543
4544         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4545             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4546             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4547             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4548
4549             # for non online change, we try to hot-unplug
4550             die "skip\n" if !$hotplug;
4551             vm_deviceunplug($vmid, $conf, $opt);
4552         } else {
4553
4554             die "internal error" if $opt !~ m/net(\d+)/;
4555             my $iface = "tap${vmid}i$1";
4556
4557             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4558                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4559                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4560                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4561                 PVE::Network::tap_unplug($iface);
4562                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4563             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4564                 # Rate can be applied on its own but any change above needs to
4565                 # include the rate in tap_plug since OVS resets everything.
4566                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4567             }
4568
4569             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4570                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4571             }
4572
4573             return 1;
4574         }
4575     }
4576
4577     if ($hotplug) {
4578         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4579     } else {
4580         die "skip\n";
4581     }
4582 }
4583
4584 sub vmconfig_update_disk {
4585     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4586
4587     # fixme: do we need force?
4588
4589     my $drive = parse_drive($opt, $value);
4590
4591     if ($conf->{$opt}) {
4592
4593         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4594
4595             my $media = $drive->{media} || 'disk';
4596             my $oldmedia = $old_drive->{media} || 'disk';
4597             die "unable to change media type\n" if $media ne $oldmedia;
4598
4599             if (!drive_is_cdrom($old_drive)) {
4600
4601                 if ($drive->{file} ne $old_drive->{file}) {
4602
4603                     die "skip\n" if !$hotplug;
4604
4605                     # unplug and register as unused
4606                     vm_deviceunplug($vmid, $conf, $opt);
4607                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4608
4609                 } else {
4610                     # update existing disk
4611
4612                     # skip non hotpluggable value
4613                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4614                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4615                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4616                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4617                         die "skip\n";
4618                     }
4619
4620                     # apply throttle
4621                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4622                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4623                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4624                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4625                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4626                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4627                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4628                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4629                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4630                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4631                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4632                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4633                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4634                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4635                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4636                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4637                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4638                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4639
4640                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4641                                                    ($drive->{mbps} || 0)*1024*1024,
4642                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4643                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4644                                                    $drive->{iops} || 0,
4645                                                    $drive->{iops_rd} || 0,
4646                                                    $drive->{iops_wr} || 0,
4647                                                    ($drive->{mbps_max} || 0)*1024*1024,
4648                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4649                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4650                                                    $drive->{iops_max} || 0,
4651                                                    $drive->{iops_rd_max} || 0,
4652                                                    $drive->{iops_wr_max} || 0,
4653                                                    $drive->{bps_max_length} || 1,
4654                                                    $drive->{bps_rd_max_length} || 1,
4655                                                    $drive->{bps_wr_max_length} || 1,
4656                                                    $drive->{iops_max_length} || 1,
4657                                                    $drive->{iops_rd_max_length} || 1,
4658                                                    $drive->{iops_wr_max_length} || 1);
4659
4660                     }
4661
4662                     return 1;
4663                 }
4664
4665             } else { # cdrom
4666
4667                 if ($drive->{file} eq 'none') {
4668                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4669                     if (drive_is_cloudinit($old_drive)) {
4670                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4671                     }
4672                 } else {
4673                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4674                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4675                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4676                 }
4677
4678                 return 1;
4679             }
4680         }
4681     }
4682
4683     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4684     # hotplug new disks
4685     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4686     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4687 }
4688
4689 sub vm_start {
4690     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4691         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4692
4693     PVE::QemuConfig->lock_config($vmid, sub {
4694         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4695
4696         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4697
4698         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4699
4700         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4701
4702         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4703             vmconfig_apply_pending($vmid, $conf, $storecfg);
4704             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4705         }
4706
4707         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4708
4709         my $defaults = load_defaults();
4710
4711         # set environment variable useful inside network script
4712         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4713
4714         my $local_volumes = {};
4715
4716         if ($targetstorage) {
4717             foreach_drive($conf, sub {
4718                 my ($ds, $drive) = @_;
4719
4720                 return if drive_is_cdrom($drive);
4721
4722                 my $volid = $drive->{file};
4723
4724                 return if !$volid;
4725
4726                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4727
4728                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4729                 return if $scfg->{shared};
4730                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4731             });
4732
4733             my $format = undef;
4734
4735             foreach my $opt (sort keys %$local_volumes) {
4736
4737                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4738                 my $drive = parse_drive($opt, $conf->{$opt});
4739
4740                 #if remote storage is specified, use default format
4741                 if ($targetstorage && $targetstorage ne "1") {
4742                     $storeid = $targetstorage;
4743                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4744                     $format = $defFormat;
4745                 } else {
4746                     #else we use same format than original
4747                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4748                     $format = qemu_img_format($scfg, $volid);
4749                 }
4750
4751                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4752                 my $newdrive = $drive;
4753                 $newdrive->{format} = $format;
4754                 $newdrive->{file} = $newvolid;
4755                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4756                 $local_volumes->{$opt} = $drivestr;
4757                 #pass drive to conf for command line
4758                 $conf->{$opt} = $drivestr;
4759             }
4760         }
4761
4762         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4763
4764         my $migrate_port = 0;
4765         my $migrate_uri;
4766         if ($statefile) {
4767             if ($statefile eq 'tcp') {
4768                 my $localip = "localhost";
4769                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4770                 my $nodename = PVE::INotify::nodename();
4771
4772                 if (!defined($migration_type)) {
4773                     if (defined($datacenterconf->{migration}->{type})) {
4774                         $migration_type = $datacenterconf->{migration}->{type};
4775                     } else {
4776                         $migration_type = 'secure';
4777                     }
4778                 }
4779
4780                 if ($migration_type eq 'insecure') {
4781                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4782                     if ($migrate_network_addr) {
4783                         $localip = $migrate_network_addr;
4784                     } else {
4785                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4786                     }
4787
4788                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4789                 }
4790
4791                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4792                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4793                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4794                 push @$cmd, '-incoming', $migrate_uri;
4795                 push @$cmd, '-S';
4796
4797             } elsif ($statefile eq 'unix') {
4798                 # should be default for secure migrations as a ssh TCP forward
4799                 # tunnel is not deterministic reliable ready and fails regurarly
4800                 # to set up in time, so use UNIX socket forwards
4801                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4802                 unlink $socket_addr;
4803
4804                 $migrate_uri = "unix:$socket_addr";
4805
4806                 push @$cmd, '-incoming', $migrate_uri;
4807                 push @$cmd, '-S';
4808
4809             } else {
4810                 push @$cmd, '-loadstate', $statefile;
4811             }
4812         } elsif ($paused) {
4813             push @$cmd, '-S';
4814         }
4815
4816         # host pci devices
4817         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4818           my $d = parse_hostpci($conf->{"hostpci$i"});
4819           next if !$d;
4820           my $pcidevices = $d->{pciid};
4821           foreach my $pcidevice (@$pcidevices) {
4822                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4823
4824                 my $info = pci_device_info("0000:$pciid");
4825                 die "IOMMU not present\n" if !check_iommu_support();
4826                 die "no pci device info for device '$pciid'\n" if !$info;
4827                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4828                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4829           }
4830         }
4831
4832         PVE::Storage::activate_volumes($storecfg, $vollist);
4833
4834         if (!check_running($vmid, 1)) {
4835             eval {
4836                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4837                     outfunc => sub {}, errfunc => sub {});
4838             };
4839         }
4840
4841         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4842                                                   : $defaults->{cpuunits};
4843
4844         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4845         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4846
4847         my %properties = (
4848             Slice => 'qemu.slice',
4849             KillMode => 'none',
4850             CPUShares => $cpuunits
4851         );
4852
4853         if (my $cpulimit = $conf->{cpulimit}) {
4854             $properties{CPUQuota} = int($cpulimit * 100);
4855         }
4856         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4857
4858         my $run_qemu = sub {
4859             PVE::Tools::run_fork sub {
4860                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4861                 run_command($cmd, %run_params);
4862             };
4863         };
4864
4865         if ($conf->{hugepages}) {
4866
4867             my $code = sub {
4868                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4869                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4870
4871                 PVE::QemuServer::Memory::hugepages_mount();
4872                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4873
4874                 eval { $run_qemu->() };
4875                 if (my $err = $@) {
4876                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4877                     die $err;
4878                 }
4879
4880                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4881             };
4882             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4883
4884         } else {
4885             eval { $run_qemu->() };
4886         }
4887
4888         if (my $err = $@) {
4889             # deactivate volumes if start fails
4890             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4891             die "start failed: $err";
4892         }
4893
4894         print "migration listens on $migrate_uri\n" if $migrate_uri;
4895
4896         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4897             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4898             warn $@ if $@;
4899         }
4900
4901         #start nbd server for storage migration
4902         if ($targetstorage) {
4903             my $nodename = PVE::INotify::nodename();
4904             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4905             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4906             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4907             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4908
4909             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4910
4911             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4912
4913             foreach my $opt (sort keys %$local_volumes) {
4914                 my $volid = $local_volumes->{$opt};
4915                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4916                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4917                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4918             }
4919         }
4920
4921         if ($migratedfrom) {
4922             eval {
4923                 set_migration_caps($vmid);
4924             };
4925             warn $@ if $@;
4926
4927             if ($spice_port) {
4928                 print "spice listens on port $spice_port\n";
4929                 if ($spice_ticket) {
4930                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4931                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4932                 }
4933             }
4934
4935         } else {
4936             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4937                 if !$statefile && $conf->{balloon};
4938
4939             foreach my $opt (keys %$conf) {
4940                 next if $opt !~  m/^net\d+$/;
4941                 my $nicconf = parse_net($conf->{$opt});
4942                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4943             }
4944         }
4945
4946         vm_mon_cmd_nocheck($vmid, 'qom-set',
4947                     path => "machine/peripheral/balloon0",
4948                     property => "guest-stats-polling-interval",
4949                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4950
4951     });
4952 }
4953
4954 sub vm_mon_cmd {
4955     my ($vmid, $execute, %params) = @_;
4956
4957     my $cmd = { execute => $execute, arguments => \%params };
4958     vm_qmp_command($vmid, $cmd);
4959 }
4960
4961 sub vm_mon_cmd_nocheck {
4962     my ($vmid, $execute, %params) = @_;
4963
4964     my $cmd = { execute => $execute, arguments => \%params };
4965     vm_qmp_command($vmid, $cmd, 1);
4966 }
4967
4968 sub vm_qmp_command {
4969     my ($vmid, $cmd, $nocheck) = @_;
4970
4971     my $res;
4972
4973     my $timeout;
4974     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4975         $timeout = $cmd->{arguments}->{timeout};
4976         delete $cmd->{arguments}->{timeout};
4977     }
4978
4979     eval {
4980         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4981         my $sname = qmp_socket($vmid);
4982         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4983             my $qmpclient = PVE::QMPClient->new();
4984
4985             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4986         } else {
4987             die "unable to open monitor socket\n";
4988         }
4989     };
4990     if (my $err = $@) {
4991         syslog("err", "VM $vmid qmp command failed - $err");
4992         die $err;
4993     }
4994
4995     return $res;
4996 }
4997
4998 sub vm_human_monitor_command {
4999     my ($vmid, $cmdline) = @_;
5000
5001     my $res;
5002
5003     my $cmd = {
5004         execute => 'human-monitor-command',
5005         arguments => { 'command-line' => $cmdline},
5006     };
5007
5008     return vm_qmp_command($vmid, $cmd);
5009 }
5010
5011 sub vm_commandline {
5012     my ($storecfg, $vmid) = @_;
5013
5014     my $conf = PVE::QemuConfig->load_config($vmid);
5015
5016     my $defaults = load_defaults();
5017
5018     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5019
5020     return PVE::Tools::cmd2string($cmd);
5021 }
5022
5023 sub vm_reset {
5024     my ($vmid, $skiplock) = @_;
5025
5026     PVE::QemuConfig->lock_config($vmid, sub {
5027
5028         my $conf = PVE::QemuConfig->load_config($vmid);
5029
5030         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5031
5032         vm_mon_cmd($vmid, "system_reset");
5033     });
5034 }
5035
5036 sub get_vm_volumes {
5037     my ($conf) = @_;
5038
5039     my $vollist = [];
5040     foreach_volid($conf, sub {
5041         my ($volid, $attr) = @_;
5042
5043         return if $volid =~ m|^/|;
5044
5045         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5046         return if !$sid;
5047
5048         push @$vollist, $volid;
5049     });
5050
5051     return $vollist;
5052 }
5053
5054 sub vm_stop_cleanup {
5055     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5056
5057     eval {
5058
5059         if (!$keepActive) {
5060             my $vollist = get_vm_volumes($conf);
5061             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5062         }
5063
5064         foreach my $ext (qw(mon qmp pid vnc qga)) {
5065             unlink "/var/run/qemu-server/${vmid}.$ext";
5066         }
5067
5068         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5069     };
5070     warn $@ if $@; # avoid errors - just warn
5071 }
5072
5073 # Note: use $nockeck to skip tests if VM configuration file exists.
5074 # We need that when migration VMs to other nodes (files already moved)
5075 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5076 sub vm_stop {
5077     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5078
5079     $force = 1 if !defined($force) && !$shutdown;
5080
5081     if ($migratedfrom){
5082         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5083         kill 15, $pid if $pid;
5084         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5085         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5086         return;
5087     }
5088
5089     PVE::QemuConfig->lock_config($vmid, sub {
5090
5091         my $pid = check_running($vmid, $nocheck);
5092         return if !$pid;
5093
5094         my $conf;
5095         if (!$nocheck) {
5096             $conf = PVE::QemuConfig->load_config($vmid);
5097             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5098             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5099                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5100                 $timeout = $opts->{down} if $opts->{down};
5101             }
5102         }
5103
5104         $timeout = 60 if !defined($timeout);
5105
5106         eval {
5107             if ($shutdown) {
5108                 if (defined($conf) && $conf->{agent}) {
5109                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5110                 } else {
5111                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5112                 }
5113             } else {
5114                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5115             }
5116         };
5117         my $err = $@;
5118
5119         if (!$err) {
5120             my $count = 0;
5121             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5122                 $count++;
5123                 sleep 1;
5124             }
5125
5126             if ($count >= $timeout) {
5127                 if ($force) {
5128                     warn "VM still running - terminating now with SIGTERM\n";
5129                     kill 15, $pid;
5130                 } else {
5131                     die "VM quit/powerdown failed - got timeout\n";
5132                 }
5133             } else {
5134                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5135                 return;
5136             }
5137         } else {
5138             if ($force) {
5139                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5140                 kill 15, $pid;
5141             } else {
5142                 die "VM quit/powerdown failed\n";
5143             }
5144         }
5145
5146         # wait again
5147         $timeout = 10;
5148
5149         my $count = 0;
5150         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5151             $count++;
5152             sleep 1;
5153         }
5154
5155         if ($count >= $timeout) {
5156             warn "VM still running - terminating now with SIGKILL\n";
5157             kill 9, $pid;
5158             sleep 1;
5159         }
5160
5161         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5162    });
5163 }
5164
5165 sub vm_suspend {
5166     my ($vmid, $skiplock) = @_;
5167
5168     PVE::QemuConfig->lock_config($vmid, sub {
5169
5170         my $conf = PVE::QemuConfig->load_config($vmid);
5171
5172         PVE::QemuConfig->check_lock($conf)
5173             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5174
5175         vm_mon_cmd($vmid, "stop");
5176     });
5177 }
5178
5179 sub vm_resume {
5180     my ($vmid, $skiplock, $nocheck) = @_;
5181
5182     PVE::QemuConfig->lock_config($vmid, sub {
5183
5184         my $res = vm_mon_cmd($vmid, 'query-status');
5185         my $resume_cmd = 'cont';
5186
5187         if ($res->{status} && $res->{status} eq 'suspended') {
5188             $resume_cmd = 'system_wakeup';
5189         }
5190
5191         if (!$nocheck) {
5192
5193             my $conf = PVE::QemuConfig->load_config($vmid);
5194
5195             PVE::QemuConfig->check_lock($conf)
5196                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5197
5198             vm_mon_cmd($vmid, $resume_cmd);
5199
5200         } else {
5201             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5202         }
5203     });
5204 }
5205
5206 sub vm_sendkey {
5207     my ($vmid, $skiplock, $key) = @_;
5208
5209     PVE::QemuConfig->lock_config($vmid, sub {
5210
5211         my $conf = PVE::QemuConfig->load_config($vmid);
5212
5213         # there is no qmp command, so we use the human monitor command
5214         vm_human_monitor_command($vmid, "sendkey $key");
5215     });
5216 }
5217
5218 sub vm_destroy {
5219     my ($storecfg, $vmid, $skiplock) = @_;
5220
5221     PVE::QemuConfig->lock_config($vmid, sub {
5222
5223         my $conf = PVE::QemuConfig->load_config($vmid);
5224
5225         if (!check_running($vmid)) {
5226             destroy_vm($storecfg, $vmid, undef, $skiplock);
5227         } else {
5228             die "VM $vmid is running - destroy failed\n";
5229         }
5230     });
5231 }
5232
5233 # pci helpers
5234
5235 sub file_write {
5236     my ($filename, $buf) = @_;
5237
5238     my $fh = IO::File->new($filename, "w");
5239     return undef if !$fh;
5240
5241     my $res = print $fh $buf;
5242
5243     $fh->close();
5244
5245     return $res;
5246 }
5247
5248 sub pci_device_info {
5249     my ($name) = @_;
5250
5251     my $res;
5252
5253     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5254     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5255
5256     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5257     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5258
5259     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5260     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5261
5262     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5263     return undef if !defined($product) || $product !~ s/^0x//;
5264
5265     $res = {
5266         name => $name,
5267         vendor => $vendor,
5268         product => $product,
5269         domain => $domain,
5270         bus => $bus,
5271         slot => $slot,
5272         func => $func,
5273         irq => $irq,
5274         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5275     };
5276
5277     return $res;
5278 }
5279
5280 sub pci_dev_reset {
5281     my ($dev) = @_;
5282
5283     my $name = $dev->{name};
5284
5285     my $fn = "$pcisysfs/devices/$name/reset";
5286
5287     return file_write($fn, "1");
5288 }
5289
5290 sub pci_dev_bind_to_vfio {
5291     my ($dev) = @_;
5292
5293     my $name = $dev->{name};
5294
5295     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5296
5297     if (!-d $vfio_basedir) {
5298         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5299     }
5300     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5301
5302     my $testdir = "$vfio_basedir/$name";
5303     return 1 if -d $testdir;
5304
5305     my $data = "$dev->{vendor} $dev->{product}";
5306     return undef if !file_write("$vfio_basedir/new_id", $data);
5307
5308     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5309     if (!file_write($fn, $name)) {
5310         return undef if -f $fn;
5311     }
5312
5313     $fn = "$vfio_basedir/bind";
5314     if (! -d $testdir) {
5315         return undef if !file_write($fn, $name);
5316     }
5317
5318     return -d $testdir;
5319 }
5320
5321 sub pci_dev_group_bind_to_vfio {
5322     my ($pciid) = @_;
5323
5324     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5325
5326     if (!-d $vfio_basedir) {
5327         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5328     }
5329     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5330
5331     # get IOMMU group devices
5332     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5333       my @devs = grep /^0000:/, readdir($D);
5334     closedir($D);
5335
5336     foreach my $pciid (@devs) {
5337         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5338
5339         # pci bridges, switches or root ports are not supported
5340         # they have a pci_bus subdirectory so skip them
5341         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5342
5343         my $info = pci_device_info($1);
5344         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5345     }
5346
5347     return 1;
5348 }
5349
5350 # vzdump restore implementaion
5351
5352 sub tar_archive_read_firstfile {
5353     my $archive = shift;
5354
5355     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5356
5357     # try to detect archive type first
5358     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5359         die "unable to open file '$archive'\n";
5360     my $firstfile = <$fh>;
5361     kill 15, $pid;
5362     close $fh;
5363
5364     die "ERROR: archive contaions no data\n" if !$firstfile;
5365     chomp $firstfile;
5366
5367     return $firstfile;
5368 }
5369
5370 sub tar_restore_cleanup {
5371     my ($storecfg, $statfile) = @_;
5372
5373     print STDERR "starting cleanup\n";
5374
5375     if (my $fd = IO::File->new($statfile, "r")) {
5376         while (defined(my $line = <$fd>)) {
5377             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5378                 my $volid = $2;
5379                 eval {
5380                     if ($volid =~ m|^/|) {
5381                         unlink $volid || die 'unlink failed\n';
5382                     } else {
5383                         PVE::Storage::vdisk_free($storecfg, $volid);
5384                     }
5385                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5386                 };
5387                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5388             } else {
5389                 print STDERR "unable to parse line in statfile - $line";
5390             }
5391         }
5392         $fd->close();
5393     }
5394 }
5395
5396 sub restore_archive {
5397     my ($archive, $vmid, $user, $opts) = @_;
5398
5399     my $format = $opts->{format};
5400     my $comp;
5401
5402     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5403         $format = 'tar' if !$format;
5404         $comp = 'gzip';
5405     } elsif ($archive =~ m/\.tar$/) {
5406         $format = 'tar' if !$format;
5407     } elsif ($archive =~ m/.tar.lzo$/) {
5408         $format = 'tar' if !$format;
5409         $comp = 'lzop';
5410     } elsif ($archive =~ m/\.vma$/) {
5411         $format = 'vma' if !$format;
5412     } elsif ($archive =~ m/\.vma\.gz$/) {
5413         $format = 'vma' if !$format;
5414         $comp = 'gzip';
5415     } elsif ($archive =~ m/\.vma\.lzo$/) {
5416         $format = 'vma' if !$format;
5417         $comp = 'lzop';
5418     } else {
5419         $format = 'vma' if !$format; # default
5420     }
5421
5422     # try to detect archive format
5423     if ($format eq 'tar') {
5424         return restore_tar_archive($archive, $vmid, $user, $opts);
5425     } else {
5426         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5427     }
5428 }
5429
5430 sub restore_update_config_line {
5431     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5432
5433     return if $line =~ m/^\#qmdump\#/;
5434     return if $line =~ m/^\#vzdump\#/;
5435     return if $line =~ m/^lock:/;
5436     return if $line =~ m/^unused\d+:/;
5437     return if $line =~ m/^parent:/;
5438     return if $line =~ m/^template:/; # restored VM is never a template
5439
5440     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5441     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5442         # try to convert old 1.X settings
5443         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5444         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5445             my ($model, $macaddr) = split(/\=/, $devconfig);
5446             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5447             my $net = {
5448                 model => $model,
5449                 bridge => "vmbr$ind",
5450                 macaddr => $macaddr,
5451             };
5452             my $netstr = print_net($net);
5453
5454             print $outfd "net$cookie->{netcount}: $netstr\n";
5455             $cookie->{netcount}++;
5456         }
5457     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5458         my ($id, $netstr) = ($1, $2);
5459         my $net = parse_net($netstr);
5460         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5461         $netstr = print_net($net);
5462         print $outfd "$id: $netstr\n";
5463     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5464         my $virtdev = $1;
5465         my $value = $3;
5466         my $di = parse_drive($virtdev, $value);
5467         if (defined($di->{backup}) && !$di->{backup}) {
5468             print $outfd "#$line";
5469         } elsif ($map->{$virtdev}) {
5470             delete $di->{format}; # format can change on restore
5471             $di->{file} = $map->{$virtdev};
5472             $value = print_drive($vmid, $di);
5473             print $outfd "$virtdev: $value\n";
5474         } else {
5475             print $outfd $line;
5476         }
5477     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5478         my ($uuid, $uuid_str);
5479         UUID::generate($uuid);
5480         UUID::unparse($uuid, $uuid_str);
5481         my $smbios1 = parse_smbios1($2);
5482         $smbios1->{uuid} = $uuid_str;
5483         print $outfd $1.print_smbios1($smbios1)."\n";
5484     } else {
5485         print $outfd $line;
5486     }
5487 }
5488
5489 sub scan_volids {
5490     my ($cfg, $vmid) = @_;
5491
5492     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5493
5494     my $volid_hash = {};
5495     foreach my $storeid (keys %$info) {
5496         foreach my $item (@{$info->{$storeid}}) {
5497             next if !($item->{volid} && $item->{size});
5498             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5499             $volid_hash->{$item->{volid}} = $item;
5500         }
5501     }
5502
5503     return $volid_hash;
5504 }
5505
5506 sub is_volume_in_use {
5507     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5508
5509     my $path = PVE::Storage::path($storecfg, $volid);
5510
5511     my $scan_config = sub {
5512         my ($cref, $snapname) = @_;
5513
5514         foreach my $key (keys %$cref) {
5515             my $value = $cref->{$key};
5516             if (is_valid_drivename($key)) {
5517                 next if $skip_drive && $key eq $skip_drive;
5518                 my $drive = parse_drive($key, $value);
5519                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5520                 return 1 if $volid eq $drive->{file};
5521                 if ($drive->{file} =~ m!^/!) {
5522                     return 1 if $drive->{file} eq $path;
5523                 } else {
5524                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5525                     next if !$storeid;
5526                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5527                     next if !$scfg;
5528                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5529                 }
5530             }
5531         }
5532
5533         return 0;
5534     };
5535
5536     return 1 if &$scan_config($conf);
5537
5538     undef $skip_drive;
5539
5540     foreach my $snapname (keys %{$conf->{snapshots}}) {
5541         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5542     }
5543
5544     return 0;
5545 }
5546
5547 sub update_disksize {
5548     my ($vmid, $conf, $volid_hash) = @_;
5549
5550     my $changes;
5551     my $prefix = "VM $vmid:";
5552
5553     # used and unused disks
5554     my $referenced = {};
5555
5556     # Note: it is allowed to define multiple storages with same path (alias), so
5557     # we need to check both 'volid' and real 'path' (two different volid can point
5558     # to the same path).
5559
5560     my $referencedpath = {};
5561
5562     # update size info
5563     foreach my $opt (keys %$conf) {
5564         if (is_valid_drivename($opt)) {
5565             my $drive = parse_drive($opt, $conf->{$opt});
5566             my $volid = $drive->{file};
5567             next if !$volid;
5568
5569             $referenced->{$volid} = 1;
5570             if ($volid_hash->{$volid} &&
5571                 (my $path = $volid_hash->{$volid}->{path})) {
5572                 $referencedpath->{$path} = 1;
5573             }
5574
5575             next if drive_is_cdrom($drive);
5576             next if !$volid_hash->{$volid};
5577
5578             $drive->{size} = $volid_hash->{$volid}->{size};
5579             my $new = print_drive($vmid, $drive);
5580             if ($new ne $conf->{$opt}) {
5581                 $changes = 1;
5582                 $conf->{$opt} = $new;
5583                 print "$prefix update disk '$opt' information.\n";
5584             }
5585         }
5586     }
5587
5588     # remove 'unusedX' entry if volume is used
5589     foreach my $opt (keys %$conf) {
5590         next if $opt !~ m/^unused\d+$/;
5591         my $volid = $conf->{$opt};
5592         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5593         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5594             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5595             $changes = 1;
5596             delete $conf->{$opt};
5597         }
5598
5599         $referenced->{$volid} = 1;
5600         $referencedpath->{$path} = 1 if $path;
5601     }
5602
5603     foreach my $volid (sort keys %$volid_hash) {
5604         next if $volid =~ m/vm-$vmid-state-/;
5605         next if $referenced->{$volid};
5606         my $path = $volid_hash->{$volid}->{path};
5607         next if !$path; # just to be sure
5608         next if $referencedpath->{$path};
5609         $changes = 1;
5610         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5611         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5612         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5613     }
5614
5615     return $changes;
5616 }
5617
5618 sub rescan {
5619     my ($vmid, $nolock, $dryrun) = @_;
5620
5621     my $cfg = PVE::Storage::config();
5622
5623     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5624     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5625     foreach my $stor (keys %{$cfg->{ids}}) {
5626         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5627     }
5628
5629     print "rescan volumes...\n";
5630     my $volid_hash = scan_volids($cfg, $vmid);
5631
5632     my $updatefn =  sub {
5633         my ($vmid) = @_;
5634
5635         my $conf = PVE::QemuConfig->load_config($vmid);
5636
5637         PVE::QemuConfig->check_lock($conf);
5638
5639         my $vm_volids = {};
5640         foreach my $volid (keys %$volid_hash) {
5641             my $info = $volid_hash->{$volid};
5642             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5643         }
5644
5645         my $changes = update_disksize($vmid, $conf, $vm_volids);
5646
5647         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5648     };
5649
5650     if (defined($vmid)) {
5651         if ($nolock) {
5652             &$updatefn($vmid);
5653         } else {
5654             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5655         }
5656     } else {
5657         my $vmlist = config_list();
5658         foreach my $vmid (keys %$vmlist) {
5659             if ($nolock) {
5660                 &$updatefn($vmid);
5661             } else {
5662                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5663             }
5664         }
5665     }
5666 }
5667
5668 sub restore_vma_archive {
5669     my ($archive, $vmid, $user, $opts, $comp) = @_;
5670
5671     my $readfrom = $archive;
5672
5673     my $cfg = PVE::Storage::config();
5674     my $commands = [];
5675     my $bwlimit = $opts->{bwlimit};
5676
5677     my $dbg_cmdstring = '';
5678     my $add_pipe = sub {
5679         my ($cmd) = @_;
5680         push @$commands, $cmd;
5681         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5682         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5683         $readfrom = '-';
5684     };
5685
5686     my $input = undef;
5687     if ($archive eq '-') {
5688         $input = '<&STDIN';
5689     } else {
5690         # If we use a backup from a PVE defined storage we also consider that
5691         # storage's rate limit:
5692         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5693         if (defined($volid)) {
5694             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5695             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5696             if ($readlimit) {
5697                 print STDERR "applying read rate limit: $readlimit\n";
5698                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5699                 $add_pipe->($cstream);
5700             }
5701         }
5702     }
5703
5704     if ($comp) {
5705         my $cmd;
5706         if ($comp eq 'gzip') {
5707             $cmd = ['zcat', $readfrom];
5708         } elsif ($comp eq 'lzop') {
5709             $cmd = ['lzop', '-d', '-c', $readfrom];
5710         } else {
5711             die "unknown compression method '$comp'\n";
5712         }
5713         $add_pipe->($cmd);
5714     }
5715
5716     my $tmpdir = "/var/tmp/vzdumptmp$$";
5717     rmtree $tmpdir;
5718
5719     # disable interrupts (always do cleanups)
5720     local $SIG{INT} =
5721         local $SIG{TERM} =
5722         local $SIG{QUIT} =
5723         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5724
5725     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5726     POSIX::mkfifo($mapfifo, 0600);
5727     my $fifofh;
5728
5729     my $openfifo = sub {
5730         open($fifofh, '>', $mapfifo) || die $!;
5731     };
5732
5733     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5734
5735     my $oldtimeout;
5736     my $timeout = 5;
5737
5738     my $devinfo = {};
5739
5740     my $rpcenv = PVE::RPCEnvironment::get();
5741
5742     my $conffile = PVE::QemuConfig->config_file($vmid);
5743     my $tmpfn = "$conffile.$$.tmp";
5744
5745     # Note: $oldconf is undef if VM does not exists
5746     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5747     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5748
5749     my %storage_limits;
5750
5751     my $print_devmap = sub {
5752         my $virtdev_hash = {};
5753
5754         my $cfgfn = "$tmpdir/qemu-server.conf";
5755
5756         # we can read the config - that is already extracted
5757         my $fh = IO::File->new($cfgfn, "r") ||
5758             "unable to read qemu-server.conf - $!\n";
5759
5760         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5761         if (-f $fwcfgfn) {
5762             my $pve_firewall_dir = '/etc/pve/firewall';
5763             mkdir $pve_firewall_dir; # make sure the dir exists
5764             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5765         }
5766
5767         while (defined(my $line = <$fh>)) {
5768             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5769                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5770                 die "archive does not contain data for drive '$virtdev'\n"
5771                     if !$devinfo->{$devname};
5772                 if (defined($opts->{storage})) {
5773                     $storeid = $opts->{storage} || 'local';
5774                 } elsif (!$storeid) {
5775                     $storeid = 'local';
5776                 }
5777                 $format = 'raw' if !$format;
5778                 $devinfo->{$devname}->{devname} = $devname;
5779                 $devinfo->{$devname}->{virtdev} = $virtdev;
5780                 $devinfo->{$devname}->{format} = $format;
5781                 $devinfo->{$devname}->{storeid} = $storeid;
5782
5783                 # check permission on storage
5784                 my $pool = $opts->{pool}; # todo: do we need that?
5785                 if ($user ne 'root@pam') {
5786                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5787                 }
5788
5789                 $storage_limits{$storeid} = $bwlimit;
5790
5791                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5792             }
5793         }
5794
5795         foreach my $key (keys %storage_limits) {
5796             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5797             next if !$limit;
5798             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5799             $storage_limits{$key} = $limit * 1024;
5800         }
5801
5802         foreach my $devname (keys %$devinfo) {
5803             die "found no device mapping information for device '$devname'\n"
5804                 if !$devinfo->{$devname}->{virtdev};
5805         }
5806
5807         # create empty/temp config
5808         if ($oldconf) {
5809             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5810             foreach_drive($oldconf, sub {
5811                 my ($ds, $drive) = @_;
5812
5813                 return if drive_is_cdrom($drive);
5814
5815                 my $volid = $drive->{file};
5816
5817                 return if !$volid || $volid =~ m|^/|;
5818
5819                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5820                 return if !$path || !$owner || ($owner != $vmid);
5821
5822                 # Note: only delete disk we want to restore
5823                 # other volumes will become unused
5824                 if ($virtdev_hash->{$ds}) {
5825                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5826                     if (my $err = $@) {
5827                         warn $err;
5828                     }
5829                 }
5830             });
5831
5832             # delete vmstate files
5833             # since after the restore we have no snapshots anymore
5834             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5835                 my $snap = $oldconf->{snapshots}->{$snapname};
5836                 if ($snap->{vmstate}) {
5837                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5838                     if (my $err = $@) {
5839                         warn $err;
5840                     }
5841                 }
5842             }
5843         }
5844
5845         my $map = {};
5846         foreach my $virtdev (sort keys %$virtdev_hash) {
5847             my $d = $virtdev_hash->{$virtdev};
5848             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5849             my $storeid = $d->{storeid};
5850             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5851
5852             my $map_opts = '';
5853             if (my $limit = $storage_limits{$storeid}) {
5854                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5855             }
5856
5857             # test if requested format is supported
5858             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5859             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5860             $d->{format} = $defFormat if !$supported;
5861
5862             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5863                                                   $d->{format}, undef, $alloc_size);
5864             print STDERR "new volume ID is '$volid'\n";
5865             $d->{volid} = $volid;
5866             my $path = PVE::Storage::path($cfg, $volid);
5867
5868             PVE::Storage::activate_volumes($cfg,[$volid]);
5869
5870             my $write_zeros = 1;
5871             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5872                 $write_zeros = 0;
5873             }
5874
5875             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5876
5877             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5878             $map->{$virtdev} = $volid;
5879         }
5880
5881         $fh->seek(0, 0) || die "seek failed - $!\n";
5882
5883         my $outfd = new IO::File ($tmpfn, "w") ||
5884             die "unable to write config for VM $vmid\n";
5885
5886         my $cookie = { netcount => 0 };
5887         while (defined(my $line = <$fh>)) {
5888             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5889         }
5890
5891         $fh->close();
5892         $outfd->close();
5893     };
5894
5895     eval {
5896         # enable interrupts
5897         local $SIG{INT} =
5898             local $SIG{TERM} =
5899             local $SIG{QUIT} =
5900             local $SIG{HUP} =
5901             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5902         local $SIG{ALRM} = sub { die "got timeout\n"; };
5903
5904         $oldtimeout = alarm($timeout);
5905
5906         my $parser = sub {
5907             my $line = shift;
5908
5909             print "$line\n";
5910
5911             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5912                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5913                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5914             } elsif ($line =~ m/^CTIME: /) {
5915                 # we correctly received the vma config, so we can disable
5916                 # the timeout now for disk allocation (set to 10 minutes, so
5917                 # that we always timeout if something goes wrong)
5918                 alarm(600);
5919                 &$print_devmap();
5920                 print $fifofh "done\n";
5921                 my $tmp = $oldtimeout || 0;
5922                 $oldtimeout = undef;
5923                 alarm($tmp);
5924                 close($fifofh);
5925             }
5926         };
5927
5928         print "restore vma archive: $dbg_cmdstring\n";
5929         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5930     };
5931     my $err = $@;
5932
5933     alarm($oldtimeout) if $oldtimeout;
5934
5935     my $vollist = [];
5936     foreach my $devname (keys %$devinfo) {
5937         my $volid = $devinfo->{$devname}->{volid};
5938         push @$vollist, $volid if $volid;
5939     }
5940
5941     PVE::Storage::deactivate_volumes($cfg, $vollist);
5942
5943     unlink $mapfifo;
5944
5945     if ($err) {
5946         rmtree $tmpdir;
5947         unlink $tmpfn;
5948
5949         foreach my $devname (keys %$devinfo) {
5950             my $volid = $devinfo->{$devname}->{volid};
5951             next if !$volid;
5952             eval {
5953                 if ($volid =~ m|^/|) {
5954                     unlink $volid || die 'unlink failed\n';
5955                 } else {
5956                     PVE::Storage::vdisk_free($cfg, $volid);
5957                 }
5958                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5959             };
5960             print STDERR "unable to cleanup '$volid' - $@" if $@;
5961         }
5962         die $err;
5963     }
5964
5965     rmtree $tmpdir;
5966
5967     rename($tmpfn, $conffile) ||
5968         die "unable to commit configuration file '$conffile'\n";
5969
5970     PVE::Cluster::cfs_update(); # make sure we read new file
5971
5972     eval { rescan($vmid, 1); };
5973     warn $@ if $@;
5974 }
5975
5976 sub restore_tar_archive {
5977     my ($archive, $vmid, $user, $opts) = @_;
5978
5979     if ($archive ne '-') {
5980         my $firstfile = tar_archive_read_firstfile($archive);
5981         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5982             if $firstfile ne 'qemu-server.conf';
5983     }
5984
5985     my $storecfg = PVE::Storage::config();
5986
5987     # destroy existing data - keep empty config
5988     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5989     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5990
5991     my $tocmd = "/usr/lib/qemu-server/qmextract";
5992
5993     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5994     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5995     $tocmd .= ' --prealloc' if $opts->{prealloc};
5996     $tocmd .= ' --info' if $opts->{info};
5997
5998     # tar option "xf" does not autodetect compression when read from STDIN,
5999     # so we pipe to zcat
6000     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6001         PVE::Tools::shellquote("--to-command=$tocmd");
6002
6003     my $tmpdir = "/var/tmp/vzdumptmp$$";
6004     mkpath $tmpdir;
6005
6006     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6007     local $ENV{VZDUMP_VMID} = $vmid;
6008     local $ENV{VZDUMP_USER} = $user;
6009
6010     my $conffile = PVE::QemuConfig->config_file($vmid);
6011     my $tmpfn = "$conffile.$$.tmp";
6012
6013     # disable interrupts (always do cleanups)
6014     local $SIG{INT} =
6015         local $SIG{TERM} =
6016         local $SIG{QUIT} =
6017         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6018
6019     eval {
6020         # enable interrupts
6021         local $SIG{INT} =
6022             local $SIG{TERM} =
6023             local $SIG{QUIT} =
6024             local $SIG{HUP} =
6025             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6026
6027         if ($archive eq '-') {
6028             print "extracting archive from STDIN\n";
6029             run_command($cmd, input => "<&STDIN");
6030         } else {
6031             print "extracting archive '$archive'\n";
6032             run_command($cmd);
6033         }
6034
6035         return if $opts->{info};
6036
6037         # read new mapping
6038         my $map = {};
6039         my $statfile = "$tmpdir/qmrestore.stat";
6040         if (my $fd = IO::File->new($statfile, "r")) {
6041             while (defined (my $line = <$fd>)) {
6042                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6043                     $map->{$1} = $2 if $1;
6044                 } else {
6045                     print STDERR "unable to parse line in statfile - $line\n";
6046                 }
6047             }
6048             $fd->close();
6049         }
6050
6051         my $confsrc = "$tmpdir/qemu-server.conf";
6052
6053         my $srcfd = new IO::File($confsrc, "r") ||
6054             die "unable to open file '$confsrc'\n";
6055
6056         my $outfd = new IO::File ($tmpfn, "w") ||
6057             die "unable to write config for VM $vmid\n";
6058
6059         my $cookie = { netcount => 0 };
6060         while (defined (my $line = <$srcfd>)) {
6061             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6062         }
6063
6064         $srcfd->close();
6065         $outfd->close();
6066     };
6067     my $err = $@;
6068
6069     if ($err) {
6070
6071         unlink $tmpfn;
6072
6073         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6074
6075         die $err;
6076     }
6077
6078     rmtree $tmpdir;
6079
6080     rename $tmpfn, $conffile ||
6081         die "unable to commit configuration file '$conffile'\n";
6082
6083     PVE::Cluster::cfs_update(); # make sure we read new file
6084
6085     eval { rescan($vmid, 1); };
6086     warn $@ if $@;
6087 };
6088
6089 sub foreach_storage_used_by_vm {
6090     my ($conf, $func) = @_;
6091
6092     my $sidhash = {};
6093
6094     foreach_drive($conf, sub {
6095         my ($ds, $drive) = @_;
6096         return if drive_is_cdrom($drive);
6097
6098         my $volid = $drive->{file};
6099
6100         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6101         $sidhash->{$sid} = $sid if $sid;
6102     });
6103
6104     foreach my $sid (sort keys %$sidhash) {
6105         &$func($sid);
6106     }
6107 }
6108
6109 sub do_snapshots_with_qemu {
6110     my ($storecfg, $volid) = @_;
6111
6112     my $storage_name = PVE::Storage::parse_volume_id($volid);
6113
6114     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6115         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6116         return 1;
6117     }
6118
6119     if ($volid =~ m/\.(qcow2|qed)$/){
6120         return 1;
6121     }
6122
6123     return undef;
6124 }
6125
6126 sub qga_check_running {
6127     my ($vmid, $nowarn) = @_;
6128
6129     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6130     if ($@) {
6131         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6132         return 0;
6133     }
6134     return 1;
6135 }
6136
6137 sub template_create {
6138     my ($vmid, $conf, $disk) = @_;
6139
6140     my $storecfg = PVE::Storage::config();
6141
6142     foreach_drive($conf, sub {
6143         my ($ds, $drive) = @_;
6144
6145         return if drive_is_cdrom($drive);
6146         return if $disk && $ds ne $disk;
6147
6148         my $volid = $drive->{file};
6149         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6150
6151         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6152         $drive->{file} = $voliddst;
6153         $conf->{$ds} = print_drive($vmid, $drive);
6154         PVE::QemuConfig->write_config($vmid, $conf);
6155     });
6156 }
6157
6158 sub qemu_img_convert {
6159     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6160
6161     my $storecfg = PVE::Storage::config();
6162     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6163     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6164
6165     if ($src_storeid && $dst_storeid) {
6166
6167         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6168
6169         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6170         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6171
6172         my $src_format = qemu_img_format($src_scfg, $src_volname);
6173         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6174
6175         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6176         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6177
6178         my $cmd = [];
6179         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6180         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6181         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6182         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6183         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6184         if ($is_zero_initialized) {
6185             push @$cmd, "zeroinit:$dst_path";
6186         } else {
6187             push @$cmd, $dst_path;
6188         }
6189
6190         my $parser = sub {
6191             my $line = shift;
6192             if($line =~ m/\((\S+)\/100\%\)/){
6193                 my $percent = $1;
6194                 my $transferred = int($size * $percent / 100);
6195                 my $remaining = $size - $transferred;
6196
6197                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6198             }
6199
6200         };
6201
6202         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6203         my $err = $@;
6204         die "copy failed: $err" if $err;
6205     }
6206 }
6207
6208 sub qemu_img_format {
6209     my ($scfg, $volname) = @_;
6210
6211     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6212         return $1;
6213     } else {
6214         return "raw";
6215     }
6216 }
6217
6218 sub qemu_drive_mirror {
6219     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6220
6221     $jobs = {} if !$jobs;
6222
6223     my $qemu_target;
6224     my $format;
6225     $jobs->{"drive-$drive"} = {};
6226
6227     if ($dst_volid =~ /^nbd:/) {
6228         $qemu_target = $dst_volid;
6229         $format = "nbd";
6230     } else {
6231         my $storecfg = PVE::Storage::config();
6232         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6233
6234         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6235
6236         $format = qemu_img_format($dst_scfg, $dst_volname);
6237
6238         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6239
6240         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6241     }
6242
6243     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6244     $opts->{format} = $format if $format;
6245
6246     print "drive mirror is starting for drive-$drive\n";
6247
6248     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6249
6250     if (my $err = $@) {
6251         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6252         die "mirroring error: $err";
6253     }
6254
6255     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6256 }
6257
6258 sub qemu_drive_mirror_monitor {
6259     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6260
6261     eval {
6262         my $err_complete = 0;
6263
6264         while (1) {
6265             die "storage migration timed out\n" if $err_complete > 300;
6266
6267             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6268
6269             my $running_mirror_jobs = {};
6270             foreach my $stat (@$stats) {
6271                 next if $stat->{type} ne 'mirror';
6272                 $running_mirror_jobs->{$stat->{device}} = $stat;
6273             }
6274
6275             my $readycounter = 0;
6276
6277             foreach my $job (keys %$jobs) {
6278
6279                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6280                     print "$job : finished\n";
6281                     delete $jobs->{$job};
6282                     next;
6283                 }
6284
6285                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6286
6287                 my $busy = $running_mirror_jobs->{$job}->{busy};
6288                 my $ready = $running_mirror_jobs->{$job}->{ready};
6289                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6290                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6291                     my $remaining = $total - $transferred;
6292                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6293
6294                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6295                 }
6296
6297                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6298             }
6299
6300             last if scalar(keys %$jobs) == 0;
6301
6302             if ($readycounter == scalar(keys %$jobs)) {
6303                 print "all mirroring jobs are ready \n";
6304                 last if $skipcomplete; #do the complete later
6305
6306                 if ($vmiddst && $vmiddst != $vmid) {
6307                     my $agent_running = $qga && qga_check_running($vmid);
6308                     if ($agent_running) {
6309                         print "freeze filesystem\n";
6310                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6311                     } else {
6312                         print "suspend vm\n";
6313                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6314                     }
6315
6316                     # if we clone a disk for a new target vm, we don't switch the disk
6317                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6318
6319                     if ($agent_running) {
6320                         print "unfreeze filesystem\n";
6321                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6322                     } else {
6323                         print "resume vm\n";
6324                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6325                     }
6326
6327                     last;
6328                 } else {
6329
6330                     foreach my $job (keys %$jobs) {
6331                         # try to switch the disk if source and destination are on the same guest
6332                         print "$job: Completing block job...\n";
6333
6334                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6335                         if ($@ =~ m/cannot be completed/) {
6336                             print "$job: Block job cannot be completed, try again.\n";
6337                             $err_complete++;
6338                         }else {
6339                             print "$job: Completed successfully.\n";
6340                             $jobs->{$job}->{complete} = 1;
6341                         }
6342                     }
6343                 }
6344             }
6345             sleep 1;
6346         }
6347     };
6348     my $err = $@;
6349
6350     if ($err) {
6351         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6352         die "mirroring error: $err";
6353     }
6354
6355 }
6356
6357 sub qemu_blockjobs_cancel {
6358     my ($vmid, $jobs) = @_;
6359
6360     foreach my $job (keys %$jobs) {
6361         print "$job: Cancelling block job\n";
6362         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6363         $jobs->{$job}->{cancel} = 1;
6364     }
6365
6366     while (1) {
6367         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6368
6369         my $running_jobs = {};
6370         foreach my $stat (@$stats) {
6371             $running_jobs->{$stat->{device}} = $stat;
6372         }
6373
6374         foreach my $job (keys %$jobs) {
6375
6376             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6377                 print "$job: Done.\n";
6378                 delete $jobs->{$job};
6379             }
6380         }
6381
6382         last if scalar(keys %$jobs) == 0;
6383
6384         sleep 1;
6385     }
6386 }
6387
6388 sub clone_disk {
6389     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6390         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6391
6392     my $newvolid;
6393
6394     if (!$full) {
6395         print "create linked clone of drive $drivename ($drive->{file})\n";
6396         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6397         push @$newvollist, $newvolid;
6398     } else {
6399
6400         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6401         $storeid = $storage if $storage;
6402
6403         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6404         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6405
6406         print "create full clone of drive $drivename ($drive->{file})\n";
6407         my $name = undef;
6408         if (drive_is_cloudinit($drive)) {
6409             $name = "vm-$newvmid-cloudinit";
6410             # cloudinit only supports raw and qcow2 atm:
6411             if ($dst_format eq 'qcow2') {
6412                 $name .= '.qcow2';
6413             } elsif ($dst_format ne 'raw') {
6414                 die "clone: unhandled format for cloudinit image\n";
6415             }
6416         }
6417         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6418         push @$newvollist, $newvolid;
6419
6420         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6421
6422         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6423         if (!$running || $snapname) {
6424             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6425         } else {
6426
6427             my $kvmver = get_running_qemu_version ($vmid);
6428             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6429                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6430                     if $drive->{iothread};
6431             }
6432
6433             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6434         }
6435     }
6436
6437     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6438
6439     my $disk = $drive;
6440     $disk->{format} = undef;
6441     $disk->{file} = $newvolid;
6442     $disk->{size} = $size;
6443
6444     return $disk;
6445 }
6446
6447 # this only works if VM is running
6448 sub get_current_qemu_machine {
6449     my ($vmid) = @_;
6450
6451     my $cmd = { execute => 'query-machines', arguments => {} };
6452     my $res = vm_qmp_command($vmid, $cmd);
6453
6454     my ($current, $default);
6455     foreach my $e (@$res) {
6456         $default = $e->{name} if $e->{'is-default'};
6457         $current = $e->{name} if $e->{'is-current'};
6458     }
6459
6460     # fallback to the default machine if current is not supported by qemu
6461     return $current || $default || 'pc';
6462 }
6463
6464 sub get_running_qemu_version {
6465     my ($vmid) = @_;
6466     my $cmd = { execute => 'query-version', arguments => {} };
6467     my $res = vm_qmp_command($vmid, $cmd);
6468     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6469 }
6470
6471 sub qemu_machine_feature_enabled {
6472     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6473
6474     my $current_major;
6475     my $current_minor;
6476
6477     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6478
6479         $current_major = $3;
6480         $current_minor = $4;
6481
6482     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6483
6484         $current_major = $1;
6485         $current_minor = $2;
6486     }
6487
6488     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6489
6490
6491 }
6492
6493 sub qemu_machine_pxe {
6494     my ($vmid, $conf, $machine) = @_;
6495
6496     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6497
6498     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6499         $machine .= '.pxe';
6500     }
6501
6502     return $machine;
6503 }
6504
6505 sub qemu_use_old_bios_files {
6506     my ($machine_type) = @_;
6507
6508     return if !$machine_type;
6509
6510     my $use_old_bios_files = undef;
6511
6512     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6513         $machine_type = $1;
6514         $use_old_bios_files = 1;
6515     } else {
6516         my $kvmver = kvm_user_version();
6517         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6518         # load new efi bios files on migration. So this hack is required to allow
6519         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6520         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6521         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6522     }
6523
6524     return ($use_old_bios_files, $machine_type);
6525 }
6526
6527 sub create_efidisk {
6528     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6529
6530     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6531
6532     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6533     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6534     PVE::Storage::activate_volumes($storecfg, [$volid]);
6535
6536     my $path = PVE::Storage::path($storecfg, $volid);
6537     eval {
6538         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6539     };
6540     die "Copying EFI vars image failed: $@" if $@;
6541
6542     return ($volid, $vars_size);
6543 }
6544
6545 sub lspci {
6546
6547     my $devices = {};
6548
6549     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6550             my (undef, $id, $function) = @_;
6551             my $res = { id => $id, function => $function};
6552             push @{$devices->{$id}}, $res;
6553     });
6554
6555     # Entries should be sorted by functions.
6556     foreach my $id (keys %$devices) {
6557         my $dev = $devices->{$id};
6558         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6559     }
6560
6561     return $devices;
6562 }
6563
6564 sub vm_iothreads_list {
6565     my ($vmid) = @_;
6566
6567     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6568
6569     my $iothreads = {};
6570     foreach my $iothread (@$res) {
6571         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6572     }
6573
6574     return $iothreads;
6575 }
6576
6577 sub scsihw_infos {
6578     my ($conf, $drive) = @_;
6579
6580     my $maxdev = 0;
6581
6582     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6583         $maxdev = 7;
6584     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6585         $maxdev = 1;
6586     } else {
6587         $maxdev = 256;
6588     }
6589
6590     my $controller = int($drive->{index} / $maxdev);
6591     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6592
6593     return ($maxdev, $controller, $controller_prefix);
6594 }
6595
6596 sub add_hyperv_enlightenments {
6597     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6598
6599     return if $winversion < 6;
6600     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6601
6602     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6603
6604     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6605         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6606         push @$cpuFlags , 'hv_vapic';
6607         push @$cpuFlags , 'hv_time';
6608     } else {
6609         push @$cpuFlags , 'hv_spinlocks=0xffff';
6610     }
6611
6612     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6613         push @$cpuFlags , 'hv_reset';
6614         push @$cpuFlags , 'hv_vpindex';
6615         push @$cpuFlags , 'hv_runtime';
6616     }
6617
6618     if ($winversion >= 7) {
6619         push @$cpuFlags , 'hv_relaxed';
6620     }
6621 }
6622
6623 sub windows_version {
6624     my ($ostype) = @_;
6625
6626     return 0 if !$ostype;
6627
6628     my $winversion = 0;
6629
6630     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6631         $winversion = 5;
6632     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6633         $winversion = 6;
6634     } elsif ($ostype =~ m/^win(\d+)$/) {
6635         $winversion = $1;
6636     }
6637
6638     return $winversion;
6639 }
6640
6641 sub resolve_dst_disk_format {
6642         my ($storecfg, $storeid, $src_volname, $format) = @_;
6643         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6644
6645         if (!$format) {
6646             # if no target format is specified, use the source disk format as hint
6647             if ($src_volname) {
6648                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6649                 $format = qemu_img_format($scfg, $src_volname);
6650             } else {
6651                 return $defFormat;
6652             }
6653         }
6654
6655         # test if requested format is supported - else use default
6656         my $supported = grep { $_ eq $format } @$validFormats;
6657         $format = $defFormat if !$supported;
6658         return $format;
6659 }
6660
6661 sub resolve_first_disk {
6662     my $conf = shift;
6663     my @disks = PVE::QemuServer::valid_drive_names();
6664     my $firstdisk;
6665     foreach my $ds (reverse @disks) {
6666         next if !$conf->{$ds};
6667         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6668         next if PVE::QemuServer::drive_is_cdrom($disk);
6669         $firstdisk = $ds;
6670     }
6671     return $firstdisk;
6672 }
6673
6674 sub generate_smbios1_uuid {
6675     my ($uuid, $uuid_str);
6676     UUID::generate($uuid);
6677     UUID::unparse($uuid, $uuid_str);
6678     return "uuid=$uuid_str";
6679 }
6680
6681 # bash completion helper
6682
6683 sub complete_backup_archives {
6684     my ($cmdname, $pname, $cvalue) = @_;
6685
6686     my $cfg = PVE::Storage::config();
6687
6688     my $storeid;
6689
6690     if ($cvalue =~ m/^([^:]+):/) {
6691         $storeid = $1;
6692     }
6693
6694     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6695
6696     my $res = [];
6697     foreach my $id (keys %$data) {
6698         foreach my $item (@{$data->{$id}}) {
6699             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6700             push @$res, $item->{volid} if defined($item->{volid});
6701         }
6702     }
6703
6704     return $res;
6705 }
6706
6707 my $complete_vmid_full = sub {
6708     my ($running) = @_;
6709
6710     my $idlist = vmstatus();
6711
6712     my $res = [];
6713
6714     foreach my $id (keys %$idlist) {
6715         my $d = $idlist->{$id};
6716         if (defined($running)) {
6717             next if $d->{template};
6718             next if $running && $d->{status} ne 'running';
6719             next if !$running && $d->{status} eq 'running';
6720         }
6721         push @$res, $id;
6722
6723     }
6724     return $res;
6725 };
6726
6727 sub complete_vmid {
6728     return &$complete_vmid_full();
6729 }
6730
6731 sub complete_vmid_stopped {
6732     return &$complete_vmid_full(0);
6733 }
6734
6735 sub complete_vmid_running {
6736     return &$complete_vmid_full(1);
6737 }
6738
6739 sub complete_storage {
6740
6741     my $cfg = PVE::Storage::config();
6742     my $ids = $cfg->{ids};
6743
6744     my $res = [];
6745     foreach my $sid (keys %$ids) {
6746         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6747         next if !$ids->{$sid}->{content}->{images};
6748         push @$res, $sid;
6749     }
6750
6751     return $res;
6752 }
6753
6754 sub nbd_stop {
6755     my ($vmid) = @_;
6756
6757     vm_mon_cmd($vmid, 'nbd-server-stop');
6758 }
6759
6760 1;