PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use PVE::Systemd;
  38 use Time::HiRes qw(gettimeofday);
  39 use File::Copy qw(copy);
  40 use URI::Escape;
  41
  42 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  43 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  44 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  45
  46 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  47
  48 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  49
  50 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  51
  52 # Note about locking: we use flock on the config file protect
  53 # against concurent actions.
  54 # Aditionaly, we have a 'lock' setting in the config file. This
  55 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  56 # allowed when such lock is set. But you can ignore this kind of
  57 # lock with the --skiplock flag.
  58
  59 cfs_register_file('/qemu-server/',
  60                   \&parse_vm_config,
  61                   \&write_vm_config);
  62
  63 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  64     description => "Some command save/restore state from this location.",
  65     type => 'string',
  66     maxLength => 128,
  67     optional => 1,
  68 });
  69
  70 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  71     description => "The name of the snapshot.",
  72     type => 'string', format => 'pve-configid',
  73     maxLength => 40,
  74 });
  75
  76 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  77     type => 'string',
  78     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  79     description => "The drive's backing file's data format.",
  80     optional => 1,
  81 });
  82
  83 #no warnings 'redefine';
  84
  85 sub cgroups_write {
  86    my ($controller, $vmid, $option, $value) = @_;
  87
  88    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  89    PVE::ProcFSTools::write_proc_entry($path, $value);
  90
  91 }
  92
  93 my $nodename = PVE::INotify::nodename();
  94
  95 mkdir "/etc/pve/nodes/$nodename";
  96 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  97 mkdir $confdir;
  98
  99 my $var_run_tmpdir = "/var/run/qemu-server";
 100 mkdir $var_run_tmpdir;
 101
 102 my $lock_dir = "/var/lock/qemu-server";
 103 mkdir $lock_dir;
 104
 105 my $pcisysfs = "/sys/bus/pci";
 106
 107 my $cpu_vendor_list = {
 108     # Intel CPUs
 109     486 => 'GenuineIntel',
 110     pentium => 'GenuineIntel',
 111     pentium2  => 'GenuineIntel',
 112     pentium3  => 'GenuineIntel',
 113     coreduo => 'GenuineIntel',
 114     core2duo => 'GenuineIntel',
 115     Conroe  => 'GenuineIntel',
 116     Penryn  => 'GenuineIntel',
 117     Nehalem  => 'GenuineIntel',
 118     'Nehalem-IBRS'  => 'GenuineIntel',
 119     Westmere => 'GenuineIntel',
 120     'Westmere-IBRS' => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     'SandyBridge-IBRS' => 'GenuineIntel',
 123     IvyBridge => 'GenuineIntel',
 124     'IvyBridge-IBRS' => 'GenuineIntel',
 125     Haswell => 'GenuineIntel',
 126     'Haswell-IBRS' => 'GenuineIntel',
 127     'Haswell-noTSX' => 'GenuineIntel',
 128     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 129     Broadwell => 'GenuineIntel',
 130     'Broadwell-IBRS' => 'GenuineIntel',
 131     'Broadwell-noTSX' => 'GenuineIntel',
 132     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 133     'Skylake-Client' => 'GenuineIntel',
 134     'Skylake-Client-IBRS' => 'GenuineIntel',
 135     'Skylake-Server' => 'GenuineIntel',
 136     'Skylake-Server-IBRS' => 'GenuineIntel',
 137
 138     # AMD CPUs
 139     athlon => 'AuthenticAMD',
 140     phenom  => 'AuthenticAMD',
 141     Opteron_G1  => 'AuthenticAMD',
 142     Opteron_G2  => 'AuthenticAMD',
 143     Opteron_G3  => 'AuthenticAMD',
 144     Opteron_G4  => 'AuthenticAMD',
 145     Opteron_G5  => 'AuthenticAMD',
 146     EPYC => 'AuthenticAMD',
 147     'EPYC-IBPB' => 'AuthenticAMD',
 148
 149     # generic types, use vendor from host node
 150     host => 'default',
 151     kvm32 => 'default',
 152     kvm64 => 'default',
 153     qemu32 => 'default',
 154     qemu64 => 'default',
 155     max => 'default',
 156 };
 157
 158 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 159
 160 my $cpu_fmt = {
 161     cputype => {
 162         description => "Emulated CPU type.",
 163         type => 'string',
 164         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 165         default => 'kvm64',
 166         default_key => 1,
 167     },
 168     hidden => {
 169         description => "Do not identify as a KVM virtual machine.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 0
 173     },
 174     flags => {
 175         description => "List of additional CPU flags separated by ';'."
 176                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 177                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 178         format_description => '+FLAG[;-FLAG...]',
 179         type => 'string',
 180         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 181         optional => 1,
 182     },
 183 };
 184
 185 my $watchdog_fmt = {
 186     model => {
 187         default_key => 1,
 188         type => 'string',
 189         enum => [qw(i6300esb ib700)],
 190         description => "Watchdog type to emulate.",
 191         default => 'i6300esb',
 192         optional => 1,
 193     },
 194     action => {
 195         type => 'string',
 196         enum => [qw(reset shutdown poweroff pause debug none)],
 197         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 198         optional => 1,
 199     },
 200 };
 201 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 202
 203 my $confdesc = {
 204     onboot => {
 205         optional => 1,
 206         type => 'boolean',
 207         description => "Specifies whether a VM will be started during system bootup.",
 208         default => 0,
 209     },
 210     autostart => {
 211         optional => 1,
 212         type => 'boolean',
 213         description => "Automatic restart after crash (currently ignored).",
 214         default => 0,
 215     },
 216     hotplug => {
 217         optional => 1,
 218         type => 'string', format => 'pve-hotplug-features',
 219         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 220         default => 'network,disk,usb',
 221     },
 222     reboot => {
 223         optional => 1,
 224         type => 'boolean',
 225         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 226         default => 1,
 227     },
 228     lock => {
 229         optional => 1,
 230         type => 'string',
 231         description => "Lock/unlock the VM.",
 232         enum => [qw(migrate backup snapshot rollback)],
 233     },
 234     cpulimit => {
 235         optional => 1,
 236         type => 'number',
 237         description => "Limit of CPU usage.",
 238         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 239         minimum => 0,
 240         maximum => 128,
 241         default => 0,
 242     },
 243     cpuunits => {
 244         optional => 1,
 245         type => 'integer',
 246         description => "CPU weight for a VM.",
 247         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 248         minimum => 2,
 249         maximum => 262144,
 250         default => 1024,
 251     },
 252     memory => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 256         minimum => 16,
 257         default => 512,
 258     },
 259     balloon => {
 260         optional => 1,
 261         type => 'integer',
 262         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 263         minimum => 0,
 264     },
 265     shares => {
 266         optional => 1,
 267         type => 'integer',
 268         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 269         minimum => 0,
 270         maximum => 50000,
 271         default => 1000,
 272     },
 273     keyboard => {
 274         optional => 1,
 275         type => 'string',
 276         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 277                        "It should not be necessary to set it.",
 278         enum => PVE::Tools::kvmkeymaplist(),
 279         default => undef,
 280     },
 281     name => {
 282         optional => 1,
 283         type => 'string', format => 'dns-name',
 284         description => "Set a name for the VM. Only used on the configuration web interface.",
 285     },
 286     scsihw => {
 287         optional => 1,
 288         type => 'string',
 289         description => "SCSI controller model",
 290         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 291         default => 'lsi',
 292     },
 293     description => {
 294         optional => 1,
 295         type => 'string',
 296         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 297     },
 298     ostype => {
 299         optional => 1,
 300         type => 'string',
 301         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 302         description => "Specify guest operating system.",
 303         verbose_description => <<EODESC,
 304 Specify guest operating system. This is used to enable special
 305 optimization/features for specific operating systems:
 306
 307 [horizontal]
 308 other;; unspecified OS
 309 wxp;; Microsoft Windows XP
 310 w2k;; Microsoft Windows 2000
 311 w2k3;; Microsoft Windows 2003
 312 w2k8;; Microsoft Windows 2008
 313 wvista;; Microsoft Windows Vista
 314 win7;; Microsoft Windows 7
 315 win8;; Microsoft Windows 8/2012/2012r2
 316 win10;; Microsoft Windows 10/2016
 317 l24;; Linux 2.4 Kernel
 318 l26;; Linux 2.6/3.X Kernel
 319 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 320 EODESC
 321     },
 322     boot => {
 323         optional => 1,
 324         type => 'string',
 325         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 326         pattern => '[acdn]{1,4}',
 327         default => 'cdn',
 328     },
 329     bootdisk => {
 330         optional => 1,
 331         type => 'string', format => 'pve-qm-bootdisk',
 332         description => "Enable booting from specified disk.",
 333         pattern => '(ide|sata|scsi|virtio)\d+',
 334     },
 335     smp => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPUs. Please use option -sockets instead.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     sockets => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of CPU sockets.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     cores => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "The number of cores per socket.",
 353         minimum => 1,
 354         default => 1,
 355     },
 356     numa => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable NUMA.",
 360         default => 0,
 361     },
 362     hugepages => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Enable/disable hugepages memory.",
 366         enum => [qw(any 2 1024)],
 367     },
 368     vcpus => {
 369         optional => 1,
 370         type => 'integer',
 371         description => "Number of hotplugged vcpus.",
 372         minimum => 1,
 373         default => 0,
 374     },
 375     acpi => {
 376         optional => 1,
 377         type => 'boolean',
 378         description => "Enable/disable ACPI.",
 379         default => 1,
 380     },
 381     agent => {
 382         optional => 1,
 383         type => 'boolean',
 384         description => "Enable/disable Qemu GuestAgent.",
 385         default => 0,
 386     },
 387     kvm => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Enable/disable KVM hardware virtualization.",
 391         default => 1,
 392     },
 393     tdf => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Enable/disable time drift fix.",
 397         default => 0,
 398     },
 399     localtime => {
 400         optional => 1,
 401         type => 'boolean',
 402         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 403     },
 404     freeze => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 408     },
 409     vga => {
 410         optional => 1,
 411         type => 'string',
 412         description => "Select the VGA type.",
 413         verbose_description => "Select the VGA type. If you want to use high resolution" .
 414             " modes (>= 1280x1024x16) then you should use the options " .
 415             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 416             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 417             "display sever. For win* OS you can select how many independent " .
 418             "displays you want, Linux guests can add displays them self. " .
 419             "You can also run without any graphic card, using a serial device" .
 420             " as terminal.",
 421         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 422     },
 423     watchdog => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-watchdog',
 426         description => "Create a virtual hardware watchdog device.",
 427         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 428             " (by a guest action), the watchdog must be periodically polled " .
 429             "by an agent inside the guest or else the watchdog will reset " .
 430             "the guest (or execute the respective action specified)",
 431     },
 432     startdate => {
 433         optional => 1,
 434         type => 'string',
 435         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 436         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 437         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 438         default => 'now',
 439     },
 440     startup =>  get_standard_option('pve-startup-order'),
 441     template => {
 442         optional => 1,
 443         type => 'boolean',
 444         description => "Enable/disable Template.",
 445         default => 0,
 446     },
 447     args => {
 448         optional => 1,
 449         type => 'string',
 450         description => "Arbitrary arguments passed to kvm.",
 451         verbose_description => <<EODESCR,
 452 Arbitrary arguments passed to kvm, for example:
 453
 454 args: -no-reboot -no-hpet
 455
 456 NOTE: this option is for experts only.
 457 EODESCR
 458     },
 459     tablet => {
 460         optional => 1,
 461         type => 'boolean',
 462         default => 1,
 463         description => "Enable/disable the USB tablet device.",
 464         verbose_description => "Enable/disable the USB tablet device. This device is " .
 465             "usually needed to allow absolute mouse positioning with VNC. " .
 466             "Else the mouse runs out of sync with normal VNC clients. " .
 467             "If you're running lots of console-only guests on one host, " .
 468             "you may consider disabling this to save some context switches. " .
 469             "This is turned off by default if you use spice (-vga=qxl).",
 470     },
 471     migrate_speed => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 475         minimum => 0,
 476         default => 0,
 477     },
 478     migrate_downtime => {
 479         optional => 1,
 480         type => 'number',
 481         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 482         minimum => 0,
 483         default => 0.1,
 484     },
 485     cdrom => {
 486         optional => 1,
 487         type => 'string', format => 'pve-qm-ide',
 488         typetext => '<volume>',
 489         description => "This is an alias for option -ide2",
 490     },
 491     cpu => {
 492         optional => 1,
 493         description => "Emulated CPU type.",
 494         type => 'string',
 495         format => $cpu_fmt,
 496     },
 497     parent => get_standard_option('pve-snapshot-name', {
 498         optional => 1,
 499         description => "Parent snapshot name. This is used internally, and should not be modified.",
 500     }),
 501     snaptime => {
 502         optional => 1,
 503         description => "Timestamp for snapshots.",
 504         type => 'integer',
 505         minimum => 0,
 506     },
 507     vmstate => {
 508         optional => 1,
 509         type => 'string', format => 'pve-volume-id',
 510         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 511     },
 512     vmstatestorage => get_standard_option('pve-storage-id', {
 513         description => "Default storage for VM state volumes/files.",
 514         optional => 1,
 515     }),
 516     machine => {
 517         description => "Specific the Qemu machine type.",
 518         type => 'string',
 519         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 520         maxLength => 40,
 521         optional => 1,
 522     },
 523     smbios1 => {
 524         description => "Specify SMBIOS type 1 fields.",
 525         type => 'string', format => 'pve-qm-smbios1',
 526         maxLength => 256,
 527         optional => 1,
 528     },
 529     protection => {
 530         optional => 1,
 531         type => 'boolean',
 532         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 533         default => 0,
 534     },
 535     bios => {
 536         optional => 1,
 537         type => 'string',
 538         enum => [ qw(seabios ovmf) ],
 539         description => "Select BIOS implementation.",
 540         default => 'seabios',
 541     },
 542 };
 543
 544 my $confdesc_cloudinit = {
 545     citype => {
 546         optional => 1,
 547         type => 'string',
 548         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 549         enum => ['configdrive2', 'nocloud'],
 550     },
 551     ciuser => {
 552         optional => 1,
 553         type => 'string',
 554         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 555     },
 556     cipassword => {
 557         optional => 1,
 558         type => 'string',
 559         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 560     },
 561     searchdomain => {
 562         optional => 1,
 563         type => 'string',
 564         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 565     },
 566     nameserver => {
 567         optional => 1,
 568         type => 'string', format => 'address-list',
 569         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 570     },
 571     sshkeys => {
 572         optional => 1,
 573         type => 'string',
 574         format => 'urlencoded',
 575         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 576     },
 577 };
 578
 579 # what about other qemu settings ?
 580 #cpu => 'string',
 581 #machine => 'string',
 582 #fda => 'file',
 583 #fdb => 'file',
 584 #mtdblock => 'file',
 585 #sd => 'file',
 586 #pflash => 'file',
 587 #snapshot => 'bool',
 588 #bootp => 'file',
 589 ##tftp => 'dir',
 590 ##smb => 'dir',
 591 #kernel => 'file',
 592 #append => 'string',
 593 #initrd => 'file',
 594 ##soundhw => 'string',
 595
 596 while (my ($k, $v) = each %$confdesc) {
 597     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 598 }
 599
 600 my $MAX_IDE_DISKS = 4;
 601 my $MAX_SCSI_DISKS = 14;
 602 my $MAX_VIRTIO_DISKS = 16;
 603 my $MAX_SATA_DISKS = 6;
 604 my $MAX_USB_DEVICES = 5;
 605 my $MAX_NETS = 32;
 606 my $MAX_UNUSED_DISKS = 8;
 607 my $MAX_HOSTPCI_DEVICES = 4;
 608 my $MAX_SERIAL_PORTS = 4;
 609 my $MAX_PARALLEL_PORTS = 3;
 610 my $MAX_NUMA = 8;
 611
 612 my $numa_fmt = {
 613     cpus => {
 614         type => "string",
 615         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 616         description => "CPUs accessing this NUMA node.",
 617         format_description => "id[-id];...",
 618     },
 619     memory => {
 620         type => "number",
 621         description => "Amount of memory this NUMA node provides.",
 622         optional => 1,
 623     },
 624     hostnodes => {
 625         type => "string",
 626         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 627         description => "Host NUMA nodes to use.",
 628         format_description => "id[-id];...",
 629         optional => 1,
 630     },
 631     policy => {
 632         type => 'string',
 633         enum => [qw(preferred bind interleave)],
 634         description => "NUMA allocation policy.",
 635         optional => 1,
 636     },
 637 };
 638 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 639 my $numadesc = {
 640     optional => 1,
 641     type => 'string', format => $numa_fmt,
 642     description => "NUMA topology.",
 643 };
 644 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 645
 646 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 647     $confdesc->{"numa$i"} = $numadesc;
 648 }
 649
 650 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 651                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 652                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 653 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 654
 655 my $net_fmt_bridge_descr = <<__EOD__;
 656 Bridge to attach the network device to. The Proxmox VE standard bridge
 657 is called 'vmbr0'.
 658
 659 If you do not specify a bridge, we create a kvm user (NATed) network
 660 device, which provides DHCP and DNS services. The following addresses
 661 are used:
 662
 663  10.0.2.2   Gateway
 664  10.0.2.3   DNS Server
 665  10.0.2.4   SMB Server
 666
 667 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 668 __EOD__
 669
 670 my $net_fmt = {
 671     macaddr => {
 672         type => 'string',
 673         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 674         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 675         format_description => "XX:XX:XX:XX:XX:XX",
 676         optional => 1,
 677     },
 678     model => {
 679         type => 'string',
 680         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 681         enum => $nic_model_list,
 682         default_key => 1,
 683     },
 684     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 685     bridge => {
 686         type => 'string',
 687         description => $net_fmt_bridge_descr,
 688         format_description => 'bridge',
 689         optional => 1,
 690     },
 691     queues => {
 692         type => 'integer',
 693         minimum => 0, maximum => 16,
 694         description => 'Number of packet queues to be used on the device.',
 695         optional => 1,
 696     },
 697     rate => {
 698         type => 'number',
 699         minimum => 0,
 700         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 701         optional => 1,
 702     },
 703     tag => {
 704         type => 'integer',
 705         minimum => 1, maximum => 4094,
 706         description => 'VLAN tag to apply to packets on this interface.',
 707         optional => 1,
 708     },
 709     trunks => {
 710         type => 'string',
 711         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 712         description => 'VLAN trunks to pass through this interface.',
 713         format_description => 'vlanid[;vlanid...]',
 714         optional => 1,
 715     },
 716     firewall => {
 717         type => 'boolean',
 718         description => 'Whether this interface should be protected by the firewall.',
 719         optional => 1,
 720     },
 721     link_down => {
 722         type => 'boolean',
 723         description => 'Whether this interface should be disconnected (like pulling the plug).',
 724         optional => 1,
 725     },
 726 };
 727
 728 my $netdesc = {
 729     optional => 1,
 730     type => 'string', format => $net_fmt,
 731     description => "Specify network devices.",
 732 };
 733
 734 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 735
 736 my $ipconfig_fmt = {
 737     ip => {
 738         type => 'string',
 739         format => 'pve-ipv4-config',
 740         format_description => 'IPv4Format/CIDR',
 741         description => 'IPv4 address in CIDR format.',
 742         optional => 1,
 743         default => 'dhcp',
 744     },
 745     gw => {
 746         type => 'string',
 747         format => 'ipv4',
 748         format_description => 'GatewayIPv4',
 749         description => 'Default gateway for IPv4 traffic.',
 750         optional => 1,
 751         requires => 'ip',
 752     },
 753     ip6 => {
 754         type => 'string',
 755         format => 'pve-ipv6-config',
 756         format_description => 'IPv6Format/CIDR',
 757         description => 'IPv6 address in CIDR format.',
 758         optional => 1,
 759         default => 'dhcp',
 760     },
 761     gw6 => {
 762         type => 'string',
 763         format => 'ipv6',
 764         format_description => 'GatewayIPv6',
 765         description => 'Default gateway for IPv6 traffic.',
 766         optional => 1,
 767         requires => 'ip6',
 768     },
 769 };
 770 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 771 my $ipconfigdesc = {
 772     optional => 1,
 773     type => 'string', format => 'pve-qm-ipconfig',
 774     description => <<'EODESCR',
 775 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 776
 777 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 778
 779 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 780 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 781
 782 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 783 EODESCR
 784 };
 785 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 786
 787 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 788     $confdesc->{"net$i"} = $netdesc;
 789     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 790 }
 791
 792 foreach my $key (keys %$confdesc_cloudinit) {
 793     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 794 }
 795
 796 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 797 sub verify_volume_id_or_qm_path {
 798     my ($volid, $noerr) = @_;
 799
 800     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 801         return $volid;
 802     }
 803
 804     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 805     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 806     if ($@) {
 807         return undef if $noerr;
 808         die $@;
 809     }
 810     return $volid;
 811 }
 812
 813 my $drivename_hash;
 814
 815 my %drivedesc_base = (
 816     volume => { alias => 'file' },
 817     file => {
 818         type => 'string',
 819         format => 'pve-volume-id-or-qm-path',
 820         default_key => 1,
 821         format_description => 'volume',
 822         description => "The drive's backing volume.",
 823     },
 824     media => {
 825         type => 'string',
 826         enum => [qw(cdrom disk)],
 827         description => "The drive's media type.",
 828         default => 'disk',
 829         optional => 1
 830     },
 831     cyls => {
 832         type => 'integer',
 833         description => "Force the drive's physical geometry to have a specific cylinder count.",
 834         optional => 1
 835     },
 836     heads => {
 837         type => 'integer',
 838         description => "Force the drive's physical geometry to have a specific head count.",
 839         optional => 1
 840     },
 841     secs => {
 842         type => 'integer',
 843         description => "Force the drive's physical geometry to have a specific sector count.",
 844         optional => 1
 845     },
 846     trans => {
 847         type => 'string',
 848         enum => [qw(none lba auto)],
 849         description => "Force disk geometry bios translation mode.",
 850         optional => 1,
 851     },
 852     snapshot => {
 853         type => 'boolean',
 854         description => "Controls qemu's snapshot mode feature."
 855             . " If activated, changes made to the disk are temporary and will"
 856             . " be discarded when the VM is shutdown.",
 857         optional => 1,
 858     },
 859     cache => {
 860         type => 'string',
 861         enum => [qw(none writethrough writeback unsafe directsync)],
 862         description => "The drive's cache mode",
 863         optional => 1,
 864     },
 865     format => get_standard_option('pve-qm-image-format'),
 866     size => {
 867         type => 'string',
 868         format => 'disk-size',
 869         format_description => 'DiskSize',
 870         description => "Disk size. This is purely informational and has no effect.",
 871         optional => 1,
 872     },
 873     backup => {
 874         type => 'boolean',
 875         description => "Whether the drive should be included when making backups.",
 876         optional => 1,
 877     },
 878     replicate => {
 879         type => 'boolean',
 880         description => 'Whether the drive should considered for replication jobs.',
 881         optional => 1,
 882         default => 1,
 883     },
 884     rerror => {
 885         type => 'string',
 886         enum => [qw(ignore report stop)],
 887         description => 'Read error action.',
 888         optional => 1,
 889     },
 890     werror => {
 891         type => 'string',
 892         enum => [qw(enospc ignore report stop)],
 893         description => 'Write error action.',
 894         optional => 1,
 895     },
 896     aio => {
 897         type => 'string',
 898         enum => [qw(native threads)],
 899         description => 'AIO type to use.',
 900         optional => 1,
 901     },
 902     discard => {
 903         type => 'string',
 904         enum => [qw(ignore on)],
 905         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 906         optional => 1,
 907     },
 908     detect_zeroes => {
 909         type => 'boolean',
 910         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 911         optional => 1,
 912     },
 913     serial => {
 914         type => 'string',
 915         format => 'urlencoded',
 916         format_description => 'serial',
 917         maxLength => 20*3, # *3 since it's %xx url enoded
 918         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 919         optional => 1,
 920     },
 921     shared => {
 922         type => 'boolean',
 923         description => 'Mark this locally-managed volume as available on all nodes',
 924         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 925         optional => 1,
 926         default => 0,
 927     }
 928 );
 929
 930 my %iothread_fmt = ( iothread => {
 931         type => 'boolean',
 932         description => "Whether to use iothreads for this drive",
 933         optional => 1,
 934 });
 935
 936 my %model_fmt = (
 937     model => {
 938         type => 'string',
 939         format => 'urlencoded',
 940         format_description => 'model',
 941         maxLength => 40*3, # *3 since it's %xx url enoded
 942         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 943         optional => 1,
 944     },
 945 );
 946
 947 my %queues_fmt = (
 948     queues => {
 949         type => 'integer',
 950         description => "Number of queues.",
 951         minimum => 2,
 952         optional => 1
 953     }
 954 );
 955
 956 my %scsiblock_fmt = (
 957     scsiblock => {
 958         type => 'boolean',
 959         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 960         optional => 1,
 961         default => 0,
 962     },
 963 );
 964
 965 my $add_throttle_desc = sub {
 966     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 967     my $d = {
 968         type => $type,
 969         format_description => $unit,
 970         description => "Maximum $what in $longunit.",
 971         optional => 1,
 972     };
 973     $d->{minimum} = $minimum if defined($minimum);
 974     $drivedesc_base{$key} = $d;
 975 };
 976 # throughput: (leaky bucket)
 977 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 979 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 980 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 982 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 983 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 984 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 985 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 986
 987 # pools: (pool of IO before throttling starts taking effect)
 988 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 990 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 991 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 992 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 993 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 994
 995 # burst lengths
 996 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 998 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1001 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1002
1003 # legacy support
1004 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1005 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1006 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1007 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1008
1009 my $ide_fmt = {
1010     %drivedesc_base,
1011     %model_fmt,
1012 };
1013 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1014
1015 my $idedesc = {
1016     optional => 1,
1017     type => 'string', format => $ide_fmt,
1018     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1019 };
1020 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1021
1022 my $scsi_fmt = {
1023     %drivedesc_base,
1024     %iothread_fmt,
1025     %queues_fmt,
1026     %scsiblock_fmt,
1027 };
1028 my $scsidesc = {
1029     optional => 1,
1030     type => 'string', format => $scsi_fmt,
1031     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1032 };
1033 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1034
1035 my $sata_fmt = {
1036     %drivedesc_base,
1037 };
1038 my $satadesc = {
1039     optional => 1,
1040     type => 'string', format => $sata_fmt,
1041     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1044
1045 my $virtio_fmt = {
1046     %drivedesc_base,
1047     %iothread_fmt,
1048 };
1049 my $virtiodesc = {
1050     optional => 1,
1051     type => 'string', format => $virtio_fmt,
1052     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1053 };
1054 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1055
1056 my $alldrive_fmt = {
1057     %drivedesc_base,
1058     %iothread_fmt,
1059     %model_fmt,
1060     %queues_fmt,
1061     %scsiblock_fmt,
1062 };
1063
1064 my $efidisk_fmt = {
1065     volume => { alias => 'file' },
1066     file => {
1067         type => 'string',
1068         format => 'pve-volume-id-or-qm-path',
1069         default_key => 1,
1070         format_description => 'volume',
1071         description => "The drive's backing volume.",
1072     },
1073     format => get_standard_option('pve-qm-image-format'),
1074     size => {
1075         type => 'string',
1076         format => 'disk-size',
1077         format_description => 'DiskSize',
1078         description => "Disk size. This is purely informational and has no effect.",
1079         optional => 1,
1080     },
1081 };
1082
1083 my $efidisk_desc = {
1084     optional => 1,
1085     type => 'string', format => $efidisk_fmt,
1086     description => "Configure a Disk for storing EFI vars",
1087 };
1088
1089 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1090
1091 my $usb_fmt = {
1092     host => {
1093         default_key => 1,
1094         type => 'string', format => 'pve-qm-usb-device',
1095         format_description => 'HOSTUSBDEVICE|spice',
1096         description => <<EODESCR,
1097 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1098
1099  'bus-port(.port)*' (decimal numbers) or
1100  'vendor_id:product_id' (hexadeciaml numbers) or
1101  'spice'
1102
1103 You can use the 'lsusb -t' command to list existing usb devices.
1104
1105 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1106
1107 The value 'spice' can be used to add a usb redirection devices for spice.
1108 EODESCR
1109     },
1110     usb3 => {
1111         optional => 1,
1112         type => 'boolean',
1113         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1114         default => 0,
1115     },
1116 };
1117
1118 my $usbdesc = {
1119     optional => 1,
1120     type => 'string', format => $usb_fmt,
1121     description => "Configure an USB device (n is 0 to 4).",
1122 };
1123 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1124
1125 # NOTE: the match-groups of this regex are used in parse_hostpci
1126 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1127 my $hostpci_fmt = {
1128     host => {
1129         default_key => 1,
1130         type => 'string',
1131         pattern => qr/$PCIRE(;$PCIRE)*/,
1132         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1133         description => <<EODESCR,
1134 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1135 of PCI virtual functions of the host. HOSTPCIID syntax is:
1136
1137 'bus:dev.func' (hexadecimal numbers)
1138
1139 You can us the 'lspci' command to list existing PCI devices.
1140 EODESCR
1141     },
1142     rombar => {
1143         type => 'boolean',
1144         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1145         optional => 1,
1146         default => 1,
1147     },
1148     romfile => {
1149         type => 'string',
1150         pattern => '[^,;]+',
1151         format_description => 'string',
1152         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1153         optional => 1,
1154     },
1155     pcie => {
1156         type => 'boolean',
1157         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1158         optional => 1,
1159         default => 0,
1160     },
1161     'x-vga' => {
1162         type => 'boolean',
1163         description =>  "Enable vfio-vga device support.",
1164         optional => 1,
1165         default => 0,
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1169
1170 my $hostpcidesc = {
1171         optional => 1,
1172         type => 'string', format => 'pve-qm-hostpci',
1173         description => "Map host PCI devices into guest.",
1174         verbose_description =>  <<EODESCR,
1175 Map host PCI devices into guest.
1176
1177 NOTE: This option allows direct access to host hardware. So it is no longer
1178 possible to migrate such machines - use with special care.
1179
1180 CAUTION: Experimental! User reported problems with this option.
1181 EODESCR
1182 };
1183 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1184
1185 my $serialdesc = {
1186         optional => 1,
1187         type => 'string',
1188         pattern => '(/dev/.+|socket)',
1189         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1190         verbose_description =>  <<EODESCR,
1191 Create a serial device inside the VM (n is 0 to 3), and pass through a
1192 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1193 host side (use 'qm terminal' to open a terminal connection).
1194
1195 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1196
1197 CAUTION: Experimental! User reported problems with this option.
1198 EODESCR
1199 };
1200
1201 my $paralleldesc= {
1202         optional => 1,
1203         type => 'string',
1204         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1205         description =>  "Map host parallel devices (n is 0 to 2).",
1206         verbose_description =>  <<EODESCR,
1207 Map host parallel devices (n is 0 to 2).
1208
1209 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1210
1211 CAUTION: Experimental! User reported problems with this option.
1212 EODESCR
1213 };
1214
1215 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1216     $confdesc->{"parallel$i"} = $paralleldesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1220     $confdesc->{"serial$i"} = $serialdesc;
1221 }
1222
1223 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1224     $confdesc->{"hostpci$i"} = $hostpcidesc;
1225 }
1226
1227 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1228     $drivename_hash->{"ide$i"} = 1;
1229     $confdesc->{"ide$i"} = $idedesc;
1230 }
1231
1232 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1233     $drivename_hash->{"sata$i"} = 1;
1234     $confdesc->{"sata$i"} = $satadesc;
1235 }
1236
1237 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1238     $drivename_hash->{"scsi$i"} = 1;
1239     $confdesc->{"scsi$i"} = $scsidesc ;
1240 }
1241
1242 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1243     $drivename_hash->{"virtio$i"} = 1;
1244     $confdesc->{"virtio$i"} = $virtiodesc;
1245 }
1246
1247 $drivename_hash->{efidisk0} = 1;
1248 $confdesc->{efidisk0} = $efidisk_desc;
1249
1250 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1251     $confdesc->{"usb$i"} = $usbdesc;
1252 }
1253
1254 my $unuseddesc = {
1255     optional => 1,
1256     type => 'string', format => 'pve-volume-id',
1257     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1258 };
1259
1260 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1261     $confdesc->{"unused$i"} = $unuseddesc;
1262 }
1263
1264 my $kvm_api_version = 0;
1265
1266 sub kvm_version {
1267
1268     return $kvm_api_version if $kvm_api_version;
1269
1270     my $fh = IO::File->new("</dev/kvm") ||
1271         return 0;
1272
1273     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1274         $kvm_api_version = $v;
1275     }
1276
1277     $fh->close();
1278
1279     return  $kvm_api_version;
1280 }
1281
1282 my $kvm_user_version;
1283
1284 sub kvm_user_version {
1285
1286     return $kvm_user_version if $kvm_user_version;
1287
1288     $kvm_user_version = 'unknown';
1289
1290     my $code = sub {
1291         my $line = shift;
1292         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1293             $kvm_user_version = $2;
1294         }
1295     };
1296
1297     eval { run_command("kvm -version", outfunc => $code); };
1298     warn $@ if $@;
1299
1300     return $kvm_user_version;
1301
1302 }
1303
1304 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1305
1306 sub valid_drive_names {
1307     # order is important - used to autoselect boot disk
1308     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1309             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1310             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1311             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1312             'efidisk0');
1313 }
1314
1315 sub is_valid_drivename {
1316     my $dev = shift;
1317
1318     return defined($drivename_hash->{$dev});
1319 }
1320
1321 sub option_exists {
1322     my $key = shift;
1323     return defined($confdesc->{$key});
1324 }
1325
1326 sub nic_models {
1327     return $nic_model_list;
1328 }
1329
1330 sub os_list_description {
1331
1332     return {
1333         other => 'Other',
1334         wxp => 'Windows XP',
1335         w2k => 'Windows 2000',
1336         w2k3 =>, 'Windows 2003',
1337         w2k8 => 'Windows 2008',
1338         wvista => 'Windows Vista',
1339         win7 => 'Windows 7',
1340         win8 => 'Windows 8/2012',
1341         win10 => 'Windows 10/2016',
1342         l24 => 'Linux 2.4',
1343         l26 => 'Linux 2.6',
1344     };
1345 }
1346
1347 my $cdrom_path;
1348
1349 sub get_cdrom_path {
1350
1351     return  $cdrom_path if $cdrom_path;
1352
1353     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1354     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1355     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1356 }
1357
1358 sub get_iso_path {
1359     my ($storecfg, $vmid, $cdrom) = @_;
1360
1361     if ($cdrom eq 'cdrom') {
1362         return get_cdrom_path();
1363     } elsif ($cdrom eq 'none') {
1364         return '';
1365     } elsif ($cdrom =~ m|^/|) {
1366         return $cdrom;
1367     } else {
1368         return PVE::Storage::path($storecfg, $cdrom);
1369     }
1370 }
1371
1372 # try to convert old style file names to volume IDs
1373 sub filename_to_volume_id {
1374     my ($vmid, $file, $media) = @_;
1375
1376      if (!($file eq 'none' || $file eq 'cdrom' ||
1377           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1378
1379         return undef if $file =~ m|/|;
1380
1381         if ($media && $media eq 'cdrom') {
1382             $file = "local:iso/$file";
1383         } else {
1384             $file = "local:$vmid/$file";
1385         }
1386     }
1387
1388     return $file;
1389 }
1390
1391 sub verify_media_type {
1392     my ($opt, $vtype, $media) = @_;
1393
1394     return if !$media;
1395
1396     my $etype;
1397     if ($media eq 'disk') {
1398         $etype = 'images';
1399     } elsif ($media eq 'cdrom') {
1400         $etype = 'iso';
1401     } else {
1402         die "internal error";
1403     }
1404
1405     return if ($vtype eq $etype);
1406
1407     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1408 }
1409
1410 sub cleanup_drive_path {
1411     my ($opt, $storecfg, $drive) = @_;
1412
1413     # try to convert filesystem paths to volume IDs
1414
1415     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1416         ($drive->{file} !~ m|^/dev/.+|) &&
1417         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1418         ($drive->{file} !~ m/^\d+$/)) {
1419         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1420         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1421         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1422         verify_media_type($opt, $vtype, $drive->{media});
1423         $drive->{file} = $volid;
1424     }
1425
1426     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1427 }
1428
1429 sub parse_hotplug_features {
1430     my ($data) = @_;
1431
1432     my $res = {};
1433
1434     return $res if $data eq '0';
1435
1436     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1437
1438     foreach my $feature (PVE::Tools::split_list($data)) {
1439         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1440             $res->{$1} = 1;
1441         } else {
1442             die "invalid hotplug feature '$feature'\n";
1443         }
1444     }
1445     return $res;
1446 }
1447
1448 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1449 sub pve_verify_hotplug_features {
1450     my ($value, $noerr) = @_;
1451
1452     return $value if parse_hotplug_features($value);
1453
1454     return undef if $noerr;
1455
1456     die "unable to parse hotplug option\n";
1457 }
1458
1459 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1460 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1461 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1462 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1463 #        [,iothread=on][,serial=serial][,model=model]
1464
1465 sub parse_drive {
1466     my ($key, $data) = @_;
1467
1468     my ($interface, $index);
1469
1470     if ($key =~ m/^([^\d]+)(\d+)$/) {
1471         $interface = $1;
1472         $index = $2;
1473     } else {
1474         return undef;
1475     }
1476
1477     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1478                                      : $confdesc->{$key}->{format};
1479     if (!$desc) {
1480         warn "invalid drive key: $key\n";
1481         return undef;
1482     }
1483     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1484     return undef if !$res;
1485     $res->{interface} = $interface;
1486     $res->{index} = $index;
1487
1488     my $error = 0;
1489     foreach my $opt (qw(bps bps_rd bps_wr)) {
1490         if (my $bps = defined(delete $res->{$opt})) {
1491             if (defined($res->{"m$opt"})) {
1492                 warn "both $opt and m$opt specified\n";
1493                 ++$error;
1494                 next;
1495             }
1496             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1497         }
1498     }
1499
1500     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1501     for my $requirement (
1502         [mbps_max => 'mbps'],
1503         [mbps_rd_max => 'mbps_rd'],
1504         [mbps_wr_max => 'mbps_wr'],
1505         [miops_max => 'miops'],
1506         [miops_rd_max => 'miops_rd'],
1507         [miops_wr_max => 'miops_wr'],
1508         [bps_max_length => 'mbps_max'],
1509         [bps_rd_max_length => 'mbps_rd_max'],
1510         [bps_wr_max_length => 'mbps_wr_max'],
1511         [iops_max_length => 'iops_max'],
1512         [iops_rd_max_length => 'iops_rd_max'],
1513         [iops_wr_max_length => 'iops_wr_max']) {
1514         my ($option, $requires) = @$requirement;
1515         if ($res->{$option} && !$res->{$requires}) {
1516             warn "$option requires $requires\n";
1517             ++$error;
1518         }
1519     }
1520
1521     return undef if $error;
1522
1523     return undef if $res->{mbps_rd} && $res->{mbps};
1524     return undef if $res->{mbps_wr} && $res->{mbps};
1525     return undef if $res->{iops_rd} && $res->{iops};
1526     return undef if $res->{iops_wr} && $res->{iops};
1527
1528     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1529         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1530         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1531         return undef if $res->{interface} eq 'virtio';
1532     }
1533
1534     if (my $size = $res->{size}) {
1535         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1536     }
1537
1538     return $res;
1539 }
1540
1541 sub print_drive {
1542     my ($vmid, $drive) = @_;
1543     my $data = { %$drive };
1544     delete $data->{$_} for qw(index interface);
1545     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1546 }
1547
1548 sub scsi_inquiry {
1549     my($fh, $noerr) = @_;
1550
1551     my $SG_IO = 0x2285;
1552     my $SG_GET_VERSION_NUM = 0x2282;
1553
1554     my $versionbuf = "\x00" x 8;
1555     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1556     if (!$ret) {
1557         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1558         return undef;
1559     }
1560     my $version = unpack("I", $versionbuf);
1561     if ($version < 30000) {
1562         die "scsi generic interface too old\n"  if !$noerr;
1563         return undef;
1564     }
1565
1566     my $buf = "\x00" x 36;
1567     my $sensebuf = "\x00" x 8;
1568     my $cmd = pack("C x3 C x1", 0x12, 36);
1569
1570     # see /usr/include/scsi/sg.h
1571     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1572
1573     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1574                       length($sensebuf), 0, length($buf), $buf,
1575                       $cmd, $sensebuf, 6000);
1576
1577     $ret = ioctl($fh, $SG_IO, $packet);
1578     if (!$ret) {
1579         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1580         return undef;
1581     }
1582
1583     my @res = unpack($sg_io_hdr_t, $packet);
1584     if ($res[17] || $res[18]) {
1585         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1586         return undef;
1587     }
1588
1589     my $res = {};
1590     (my $byte0, my $byte1, $res->{vendor},
1591      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1592
1593     $res->{removable} = $byte1 & 128 ? 1 : 0;
1594     $res->{type} = $byte0 & 31;
1595
1596     return $res;
1597 }
1598
1599 sub path_is_scsi {
1600     my ($path) = @_;
1601
1602     my $fh = IO::File->new("+<$path") || return undef;
1603     my $res = scsi_inquiry($fh, 1);
1604     close($fh);
1605
1606     return $res;
1607 }
1608
1609 sub machine_type_is_q35 {
1610     my ($conf) = @_;
1611
1612     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1613 }
1614
1615 sub print_tabletdevice_full {
1616     my ($conf) = @_;
1617
1618     my $q35 = machine_type_is_q35($conf);
1619
1620     # we use uhci for old VMs because tablet driver was buggy in older qemu
1621     my $usbbus = $q35 ? "ehci" : "uhci";
1622
1623     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1624 }
1625
1626 sub print_drivedevice_full {
1627     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1628
1629     my $device = '';
1630     my $maxdev = 0;
1631
1632     if ($drive->{interface} eq 'virtio') {
1633         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1634         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1635         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1636     } elsif ($drive->{interface} eq 'scsi') {
1637
1638         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1639         my $unit = $drive->{index} % $maxdev;
1640         my $devicetype = 'hd';
1641         my $path = '';
1642         if (drive_is_cdrom($drive)) {
1643             $devicetype = 'cd';
1644         } else {
1645             if ($drive->{file} =~ m|^/|) {
1646                 $path = $drive->{file};
1647                 if (my $info = path_is_scsi($path)) {
1648                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1649                         $devicetype = 'block';
1650                     } elsif ($info->{type} == 1) { # tape
1651                         $devicetype = 'generic';
1652                     }
1653                 }
1654             } else {
1655                  $path = PVE::Storage::path($storecfg, $drive->{file});
1656             }
1657
1658             if($path =~ m/^iscsi\:\/\//){
1659                 $devicetype = 'generic';
1660             }
1661         }
1662
1663         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         } else {
1666             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1667         }
1668
1669     } elsif ($drive->{interface} eq 'ide'){
1670         $maxdev = 2;
1671         my $controller = int($drive->{index} / $maxdev);
1672         my $unit = $drive->{index} % $maxdev;
1673         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1674
1675         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1676         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1677             $model = URI::Escape::uri_unescape($model);
1678             $device .= ",model=$model";
1679         }
1680     } elsif ($drive->{interface} eq 'sata'){
1681         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1682         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1683         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1684     } elsif ($drive->{interface} eq 'usb') {
1685         die "implement me";
1686         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1687     } else {
1688         die "unsupported interface type";
1689     }
1690
1691     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1692
1693     if (my $serial = $drive->{serial}) {
1694         $serial = URI::Escape::uri_unescape($serial);
1695         $device .= ",serial=$serial";
1696     }
1697
1698
1699     return $device;
1700 }
1701
1702 sub get_initiator_name {
1703     my $initiator;
1704
1705     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1706     while (defined(my $line = <$fh>)) {
1707         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1708         $initiator = $1;
1709         last;
1710     }
1711     $fh->close();
1712
1713     return $initiator;
1714 }
1715
1716 sub print_drive_full {
1717     my ($storecfg, $vmid, $drive) = @_;
1718
1719     my $path;
1720     my $volid = $drive->{file};
1721     my $format;
1722
1723     if (drive_is_cdrom($drive)) {
1724         $path = get_iso_path($storecfg, $vmid, $volid);
1725     } else {
1726         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1727         if ($storeid) {
1728             $path = PVE::Storage::path($storecfg, $volid);
1729             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1730             $format = qemu_img_format($scfg, $volname);
1731         } else {
1732             $path = $volid;
1733             $format = "raw";
1734         }
1735    }
1736
1737     my $opts = '';
1738     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1739     foreach my $o (@qemu_drive_options) {
1740         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1741     }
1742
1743     # snapshot only accepts on|off
1744     if (defined($drive->{snapshot})) {
1745         my $v = $drive->{snapshot} ? 'on' : 'off';
1746         $opts .= ",snapshot=$v";
1747     }
1748
1749     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1750         my ($dir, $qmpname) = @$type;
1751         if (my $v = $drive->{"mbps$dir"}) {
1752             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1753         }
1754         if (my $v = $drive->{"mbps${dir}_max"}) {
1755             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1756         }
1757         if (my $v = $drive->{"bps${dir}_max_length"}) {
1758             $opts .= ",throttling.bps$qmpname-max-length=$v";
1759         }
1760         if (my $v = $drive->{"iops${dir}"}) {
1761             $opts .= ",throttling.iops$qmpname=$v";
1762         }
1763         if (my $v = $drive->{"iops${dir}_max"}) {
1764             $opts .= ",throttling.iops$qmpname-max=$v";
1765         }
1766         if (my $v = $drive->{"iops${dir}_max_length"}) {
1767             $opts .= ",throttling.iops$qmpname-max-length=$v";
1768         }
1769     }
1770
1771     $opts .= ",format=$format" if $format && !$drive->{format};
1772
1773     my $cache_direct = 0;
1774
1775     if (my $cache = $drive->{cache}) {
1776         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1777     } elsif (!drive_is_cdrom($drive)) {
1778         $opts .= ",cache=none";
1779         $cache_direct = 1;
1780     }
1781
1782     # aio native works only with O_DIRECT
1783     if (!$drive->{aio}) {
1784         if($cache_direct) {
1785             $opts .= ",aio=native";
1786         } else {
1787             $opts .= ",aio=threads";
1788         }
1789     }
1790
1791     if (!drive_is_cdrom($drive)) {
1792         my $detectzeroes;
1793         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1794             $detectzeroes = 'off';
1795         } elsif ($drive->{discard}) {
1796             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1797         } else {
1798             # This used to be our default with discard not being specified:
1799             $detectzeroes = 'on';
1800         }
1801         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1802     }
1803
1804     my $pathinfo = $path ? "file=$path," : '';
1805
1806     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1807 }
1808
1809 sub print_netdevice_full {
1810     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1811
1812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1813
1814     my $device = $net->{model};
1815     if ($net->{model} eq 'virtio') {
1816          $device = 'virtio-net-pci';
1817      };
1818
1819     my $pciaddr = print_pci_addr("$netid", $bridges);
1820     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1821     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1822         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1823         my $vectors = $net->{queues} * 2 + 2;
1824         $tmpstr .= ",vectors=$vectors,mq=on";
1825     }
1826     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1827
1828     if ($use_old_bios_files) {
1829         my $romfile;
1830         if ($device eq 'virtio-net-pci') {
1831             $romfile = 'pxe-virtio.rom';
1832         } elsif ($device eq 'e1000') {
1833             $romfile = 'pxe-e1000.rom';
1834         } elsif ($device eq 'ne2k') {
1835             $romfile = 'pxe-ne2k_pci.rom';
1836         } elsif ($device eq 'pcnet') {
1837             $romfile = 'pxe-pcnet.rom';
1838         } elsif ($device eq 'rtl8139') {
1839             $romfile = 'pxe-rtl8139.rom';
1840         }
1841         $tmpstr .= ",romfile=$romfile" if $romfile;
1842     }
1843
1844     return $tmpstr;
1845 }
1846
1847 sub print_netdev_full {
1848     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1849
1850     my $i = '';
1851     if ($netid =~ m/^net(\d+)$/) {
1852         $i = int($1);
1853     }
1854
1855     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1856
1857     my $ifname = "tap${vmid}i$i";
1858
1859     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1860     die "interface name '$ifname' is too long (max 15 character)\n"
1861         if length($ifname) >= 16;
1862
1863     my $vhostparam = '';
1864     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1865
1866     my $vmname = $conf->{name} || "vm$vmid";
1867
1868     my $netdev = "";
1869     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1870
1871     if ($net->{bridge}) {
1872         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1873     } else {
1874         $netdev = "type=user,id=$netid,hostname=$vmname";
1875     }
1876
1877     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1878
1879     return $netdev;
1880 }
1881
1882
1883 sub print_cpu_device {
1884     my ($conf, $id) = @_;
1885
1886     my $kvm = $conf->{kvm} // 1;
1887     my $cpu = $kvm ? "kvm64" : "qemu64";
1888     if (my $cputype = $conf->{cpu}) {
1889         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1890             or die "Cannot parse cpu description: $cputype\n";
1891         $cpu = $cpuconf->{cputype};
1892     }
1893
1894     my $cores = $conf->{cores} || 1;
1895
1896     my $current_core = ($id - 1) % $cores;
1897     my $current_socket = int(($id - 1 - $current_core)/$cores);
1898
1899     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1900 }
1901
1902 sub drive_is_cloudinit {
1903     my ($drive) = @_;
1904     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1905 }
1906
1907 sub drive_is_cdrom {
1908     my ($drive, $exclude_cloudinit) = @_;
1909
1910     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1911
1912     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1913
1914 }
1915
1916 sub parse_number_sets {
1917     my ($set) = @_;
1918     my $res = [];
1919     foreach my $part (split(/;/, $set)) {
1920         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1921             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1922             push @$res, [ $1, $2 ];
1923         } else {
1924             die "invalid range: $part\n";
1925         }
1926     }
1927     return $res;
1928 }
1929
1930 sub parse_numa {
1931     my ($data) = @_;
1932
1933     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1934     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1935     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1936     return $res;
1937 }
1938
1939 sub parse_hostpci {
1940     my ($value) = @_;
1941
1942     return undef if !$value;
1943
1944     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1945
1946     my @idlist = split(/;/, $res->{host});
1947     delete $res->{host};
1948     foreach my $id (@idlist) {
1949         if ($id =~ /^$PCIRE$/) {
1950             if (defined($2)) {
1951                 push @{$res->{pciid}}, { id => $1, function => $2 };
1952             } else {
1953                 my $pcidevices = lspci($1);
1954                 $res->{pciid} = $pcidevices->{$1};
1955             }
1956         } else {
1957             # should have been caught by parse_property_string already
1958             die "failed to parse PCI id: $id\n";
1959         }
1960     }
1961     return $res;
1962 }
1963
1964 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1965 sub parse_net {
1966     my ($data) = @_;
1967
1968     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1969     if ($@) {
1970         warn $@;
1971         return undef;
1972     }
1973     if (!defined($res->{macaddr})) {
1974         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1975         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1976     }
1977     return $res;
1978 }
1979
1980 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1981 sub parse_ipconfig {
1982     my ($data) = @_;
1983
1984     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return undef;
1988     }
1989
1990     if ($res->{gw} && !$res->{ip}) {
1991         warn 'gateway specified without specifying an IP address';
1992         return undef;
1993     }
1994     if ($res->{gw6} && !$res->{ip6}) {
1995         warn 'IPv6 gateway specified without specifying an IPv6 address';
1996         return undef;
1997     }
1998     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1999         warn 'gateway specified together with DHCP';
2000         return undef;
2001     }
2002     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2003         # gw6 + auto/dhcp
2004         warn "IPv6 gateway specified together with $res->{ip6} address";
2005         return undef;
2006     }
2007
2008     if (!$res->{ip} && !$res->{ip6}) {
2009         return { ip => 'dhcp', ip6 => 'dhcp' };
2010     }
2011
2012     return $res;
2013 }
2014
2015 sub print_net {
2016     my $net = shift;
2017
2018     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2019 }
2020
2021 sub add_random_macs {
2022     my ($settings) = @_;
2023
2024     foreach my $opt (keys %$settings) {
2025         next if $opt !~ m/^net(\d+)$/;
2026         my $net = parse_net($settings->{$opt});
2027         next if !$net;
2028         $settings->{$opt} = print_net($net);
2029     }
2030 }
2031
2032 sub vm_is_volid_owner {
2033     my ($storecfg, $vmid, $volid) = @_;
2034
2035     if ($volid !~  m|^/|) {
2036         my ($path, $owner);
2037         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2038         if ($owner && ($owner == $vmid)) {
2039             return 1;
2040         }
2041     }
2042
2043     return undef;
2044 }
2045
2046 sub split_flagged_list {
2047     my $text = shift || '';
2048     $text =~ s/[,;]/ /g;
2049     $text =~ s/^\s+//;
2050     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2051 }
2052
2053 sub join_flagged_list {
2054     my ($how, $lst) = @_;
2055     join $how, map { $lst->{$_} . $_ } keys %$lst;
2056 }
2057
2058 sub vmconfig_delete_pending_option {
2059     my ($conf, $key, $force) = @_;
2060
2061     delete $conf->{pending}->{$key};
2062     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2063     $pending_delete_hash->{$key} = $force ? '!' : '';
2064     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2065 }
2066
2067 sub vmconfig_undelete_pending_option {
2068     my ($conf, $key) = @_;
2069
2070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2071     delete $pending_delete_hash->{$key};
2072
2073     if (%$pending_delete_hash) {
2074         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2075     } else {
2076         delete $conf->{pending}->{delete};
2077     }
2078 }
2079
2080 sub vmconfig_register_unused_drive {
2081     my ($storecfg, $vmid, $conf, $drive) = @_;
2082
2083     if (drive_is_cloudinit($drive)) {
2084         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2085         warn $@ if $@;
2086     } elsif (!drive_is_cdrom($drive)) {
2087         my $volid = $drive->{file};
2088         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2089             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2090         }
2091     }
2092 }
2093
2094 sub vmconfig_cleanup_pending {
2095     my ($conf) = @_;
2096
2097     # remove pending changes when nothing changed
2098     my $changes;
2099     foreach my $opt (keys %{$conf->{pending}}) {
2100         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2101             $changes = 1;
2102             delete $conf->{pending}->{$opt};
2103         }
2104     }
2105
2106     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2107     my $pending_delete_hash = {};
2108     while (my ($opt, $force) = each %$current_delete_hash) {
2109         if (defined($conf->{$opt})) {
2110             $pending_delete_hash->{$opt} = $force;
2111         } else {
2112             $changes = 1;
2113         }
2114     }
2115
2116     if (%$pending_delete_hash) {
2117         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2118     } else {
2119         delete $conf->{pending}->{delete};
2120     }
2121
2122     return $changes;
2123 }
2124
2125 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2126 my $smbios1_fmt = {
2127     uuid => {
2128         type => 'string',
2129         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2130         format_description => 'UUID',
2131         description => "Set SMBIOS1 UUID.",
2132         optional => 1,
2133     },
2134     version => {
2135         type => 'string',
2136         pattern => '\S+',
2137         format_description => 'string',
2138         description => "Set SMBIOS1 version.",
2139         optional => 1,
2140     },
2141     serial => {
2142         type => 'string',
2143         pattern => '\S+',
2144         format_description => 'string',
2145         description => "Set SMBIOS1 serial number.",
2146         optional => 1,
2147     },
2148     manufacturer => {
2149         type => 'string',
2150         pattern => '\S+',
2151         format_description => 'string',
2152         description => "Set SMBIOS1 manufacturer.",
2153         optional => 1,
2154     },
2155     product => {
2156         type => 'string',
2157         pattern => '\S+',
2158         format_description => 'string',
2159         description => "Set SMBIOS1 product ID.",
2160         optional => 1,
2161     },
2162     sku => {
2163         type => 'string',
2164         pattern => '\S+',
2165         format_description => 'string',
2166         description => "Set SMBIOS1 SKU string.",
2167         optional => 1,
2168     },
2169     family => {
2170         type => 'string',
2171         pattern => '\S+',
2172         format_description => 'string',
2173         description => "Set SMBIOS1 family string.",
2174         optional => 1,
2175     },
2176 };
2177
2178 sub parse_smbios1 {
2179     my ($data) = @_;
2180
2181     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2182     warn $@ if $@;
2183     return $res;
2184 }
2185
2186 sub print_smbios1 {
2187     my ($smbios1) = @_;
2188     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2192
2193 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2194 sub verify_bootdisk {
2195     my ($value, $noerr) = @_;
2196
2197     return $value if is_valid_drivename($value);
2198
2199     return undef if $noerr;
2200
2201     die "invalid boot disk '$value'\n";
2202 }
2203
2204 sub parse_watchdog {
2205     my ($value) = @_;
2206
2207     return undef if !$value;
2208
2209     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2210     warn $@ if $@;
2211     return $res;
2212 }
2213
2214 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2215 sub verify_usb_device {
2216     my ($value, $noerr) = @_;
2217
2218     return $value if parse_usb_device($value);
2219
2220     return undef if $noerr;
2221
2222     die "unable to parse usb device\n";
2223 }
2224
2225 # add JSON properties for create and set function
2226 sub json_config_properties {
2227     my $prop = shift;
2228
2229     foreach my $opt (keys %$confdesc) {
2230         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2231         $prop->{$opt} = $confdesc->{$opt};
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # return copy of $confdesc_cloudinit to generate documentation
2238 sub cloudinit_config_properties {
2239
2240     return dclone($confdesc_cloudinit);
2241 }
2242
2243 sub check_type {
2244     my ($key, $value) = @_;
2245
2246     die "unknown setting '$key'\n" if !$confdesc->{$key};
2247
2248     my $type = $confdesc->{$key}->{type};
2249
2250     if (!defined($value)) {
2251         die "got undefined value\n";
2252     }
2253
2254     if ($value =~ m/[\n\r]/) {
2255         die "property contains a line feed\n";
2256     }
2257
2258     if ($type eq 'boolean') {
2259         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2260         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2261         die "type check ('boolean') failed - got '$value'\n";
2262     } elsif ($type eq 'integer') {
2263         return int($1) if $value =~ m/^(\d+)$/;
2264         die "type check ('integer') failed - got '$value'\n";
2265     } elsif ($type eq 'number') {
2266         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2267         die "type check ('number') failed - got '$value'\n";
2268     } elsif ($type eq 'string') {
2269         if (my $fmt = $confdesc->{$key}->{format}) {
2270             PVE::JSONSchema::check_format($fmt, $value);
2271             return $value;
2272         }
2273         $value =~ s/^\"(.*)\"$/$1/;
2274         return $value;
2275     } else {
2276         die "internal error"
2277     }
2278 }
2279
2280 sub check_iommu_support{
2281     #fixme : need to check IOMMU support
2282     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2283
2284     my $iommu=1;
2285     return $iommu;
2286
2287 }
2288
2289 sub touch_config {
2290     my ($vmid) = @_;
2291
2292     my $conf = PVE::QemuConfig->config_file($vmid);
2293     utime undef, undef, $conf;
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2298
2299     my $conffile = PVE::QemuConfig->config_file($vmid);
2300
2301     my $conf = PVE::QemuConfig->load_config($vmid);
2302
2303     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2304
2305     if ($conf->{template}) {
2306         # check if any base image is still used by a linked clone
2307         foreach_drive($conf, sub {
2308                 my ($ds, $drive) = @_;
2309
2310                 return if drive_is_cdrom($drive);
2311
2312                 my $volid = $drive->{file};
2313
2314                 return if !$volid || $volid =~ m|^/|;
2315
2316                 die "base volume '$volid' is still in use by linked cloned\n"
2317                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2318
2319         });
2320     }
2321
2322     # only remove disks owned by this VM
2323     foreach_drive($conf, sub {
2324         my ($ds, $drive) = @_;
2325
2326         return if drive_is_cdrom($drive, 1);
2327
2328         my $volid = $drive->{file};
2329
2330         return if !$volid || $volid =~ m|^/|;
2331
2332         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2333         return if !$path || !$owner || ($owner != $vmid);
2334
2335         eval {
2336             PVE::Storage::vdisk_free($storecfg, $volid);
2337         };
2338         warn "Could not remove disk '$volid', check manually: $@" if $@;
2339
2340     });
2341
2342     if ($keep_empty_config) {
2343         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2344     } else {
2345         unlink $conffile;
2346     }
2347
2348     # also remove unused disk
2349     eval {
2350         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2351
2352         eval {
2353             PVE::Storage::foreach_volid($dl, sub {
2354                 my ($volid, $sid, $volname, $d) = @_;
2355                 PVE::Storage::vdisk_free($storecfg, $volid);
2356             });
2357         };
2358         warn $@ if $@;
2359
2360     };
2361     warn $@ if $@;
2362 }
2363
2364 sub parse_vm_config {
2365     my ($filename, $raw) = @_;
2366
2367     return undef if !defined($raw);
2368
2369     my $res = {
2370         digest => Digest::SHA::sha1_hex($raw),
2371         snapshots => {},
2372         pending => {},
2373     };
2374
2375     $filename =~ m|/qemu-server/(\d+)\.conf$|
2376         || die "got strange filename '$filename'";
2377
2378     my $vmid = $1;
2379
2380     my $conf = $res;
2381     my $descr;
2382     my $section = '';
2383
2384     my @lines = split(/\n/, $raw);
2385     foreach my $line (@lines) {
2386         next if $line =~ m/^\s*$/;
2387
2388         if ($line =~ m/^\[PENDING\]\s*$/i) {
2389             $section = 'pending';
2390             if (defined($descr)) {
2391                 $descr =~ s/\s+$//;
2392                 $conf->{description} = $descr;
2393             }
2394             $descr = undef;
2395             $conf = $res->{$section} = {};
2396             next;
2397
2398         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2399             $section = $1;
2400             if (defined($descr)) {
2401                 $descr =~ s/\s+$//;
2402                 $conf->{description} = $descr;
2403             }
2404             $descr = undef;
2405             $conf = $res->{snapshots}->{$section} = {};
2406             next;
2407         }
2408
2409         if ($line =~ m/^\#(.*)\s*$/) {
2410             $descr = '' if !defined($descr);
2411             $descr .= PVE::Tools::decode_text($1) . "\n";
2412             next;
2413         }
2414
2415         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2416             $descr = '' if !defined($descr);
2417             $descr .= PVE::Tools::decode_text($2);
2418         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2419             $conf->{snapstate} = $1;
2420         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2421             my $key = $1;
2422             my $value = $2;
2423             $conf->{$key} = $value;
2424         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2425             my $value = $1;
2426             if ($section eq 'pending') {
2427                 $conf->{delete} = $value; # we parse this later
2428             } else {
2429                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2430             }
2431         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2432             my $key = $1;
2433             my $value = $2;
2434             eval { $value = check_type($key, $value); };
2435             if ($@) {
2436                 warn "vm $vmid - unable to parse value of '$key' - $@";
2437             } else {
2438                 $key = 'ide2' if $key eq 'cdrom';
2439                 my $fmt = $confdesc->{$key}->{format};
2440                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2441                     my $v = parse_drive($key, $value);
2442                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2443                         $v->{file} = $volid;
2444                         $value = print_drive($vmid, $v);
2445                     } else {
2446                         warn "vm $vmid - unable to parse value of '$key'\n";
2447                         next;
2448                     }
2449                 }
2450
2451                 $conf->{$key} = $value;
2452             }
2453         }
2454     }
2455
2456     if (defined($descr)) {
2457         $descr =~ s/\s+$//;
2458         $conf->{description} = $descr;
2459     }
2460     delete $res->{snapstate}; # just to be sure
2461
2462     return $res;
2463 }
2464
2465 sub write_vm_config {
2466     my ($filename, $conf) = @_;
2467
2468     delete $conf->{snapstate}; # just to be sure
2469
2470     if ($conf->{cdrom}) {
2471         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2472         $conf->{ide2} = $conf->{cdrom};
2473         delete $conf->{cdrom};
2474     }
2475
2476     # we do not use 'smp' any longer
2477     if ($conf->{sockets}) {
2478         delete $conf->{smp};
2479     } elsif ($conf->{smp}) {
2480         $conf->{sockets} = $conf->{smp};
2481         delete $conf->{cores};
2482         delete $conf->{smp};
2483     }
2484
2485     my $used_volids = {};
2486
2487     my $cleanup_config = sub {
2488         my ($cref, $pending, $snapname) = @_;
2489
2490         foreach my $key (keys %$cref) {
2491             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2492                 $key eq 'snapstate' || $key eq 'pending';
2493             my $value = $cref->{$key};
2494             if ($key eq 'delete') {
2495                 die "propertry 'delete' is only allowed in [PENDING]\n"
2496                     if !$pending;
2497                 # fixme: check syntax?
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             die "unable to parse value of '$key' - $@" if $@;
2502
2503             $cref->{$key} = $value;
2504
2505             if (!$snapname && is_valid_drivename($key)) {
2506                 my $drive = parse_drive($key, $value);
2507                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2508             }
2509         }
2510     };
2511
2512     &$cleanup_config($conf);
2513
2514     &$cleanup_config($conf->{pending}, 1);
2515
2516     foreach my $snapname (keys %{$conf->{snapshots}}) {
2517         die "internal error" if $snapname eq 'pending';
2518         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2519     }
2520
2521     # remove 'unusedX' settings if we re-add a volume
2522     foreach my $key (keys %$conf) {
2523         my $value = $conf->{$key};
2524         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2525             delete $conf->{$key};
2526         }
2527     }
2528
2529     my $generate_raw_config = sub {
2530         my ($conf, $pending) = @_;
2531
2532         my $raw = '';
2533
2534         # add description as comment to top of file
2535         if (defined(my $descr = $conf->{description})) {
2536             if ($descr) {
2537                 foreach my $cl (split(/\n/, $descr)) {
2538                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2539                 }
2540             } else {
2541                 $raw .= "#\n" if $pending;
2542             }
2543         }
2544
2545         foreach my $key (sort keys %$conf) {
2546             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2547             $raw .= "$key: $conf->{$key}\n";
2548         }
2549         return $raw;
2550     };
2551
2552     my $raw = &$generate_raw_config($conf);
2553
2554     if (scalar(keys %{$conf->{pending}})){
2555         $raw .= "\n[PENDING]\n";
2556         $raw .= &$generate_raw_config($conf->{pending}, 1);
2557     }
2558
2559     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2560         $raw .= "\n[$snapname]\n";
2561         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2562     }
2563
2564     return $raw;
2565 }
2566
2567 sub load_defaults {
2568
2569     my $res = {};
2570
2571     # we use static defaults from our JSON schema configuration
2572     foreach my $key (keys %$confdesc) {
2573         if (defined(my $default = $confdesc->{$key}->{default})) {
2574             $res->{$key} = $default;
2575         }
2576     }
2577
2578     return $res;
2579 }
2580
2581 sub config_list {
2582     my $vmlist = PVE::Cluster::get_vmlist();
2583     my $res = {};
2584     return $res if !$vmlist || !$vmlist->{ids};
2585     my $ids = $vmlist->{ids};
2586
2587     foreach my $vmid (keys %$ids) {
2588         my $d = $ids->{$vmid};
2589         next if !$d->{node} || $d->{node} ne $nodename;
2590         next if !$d->{type} || $d->{type} ne 'qemu';
2591         $res->{$vmid}->{exists} = 1;
2592     }
2593     return $res;
2594 }
2595
2596 # test if VM uses local resources (to prevent migration)
2597 sub check_local_resources {
2598     my ($conf, $noerr) = @_;
2599
2600     my $loc_res = 0;
2601
2602     $loc_res = 1 if $conf->{hostusb}; # old syntax
2603     $loc_res = 1 if $conf->{hostpci}; # old syntax
2604
2605     foreach my $k (keys %$conf) {
2606         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2607         # sockets are safe: they will recreated be on the target side post-migrate
2608         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2609         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2610     }
2611
2612     die "VM uses local resources\n" if $loc_res && !$noerr;
2613
2614     return $loc_res;
2615 }
2616
2617 # check if used storages are available on all nodes (use by migrate)
2618 sub check_storage_availability {
2619     my ($storecfg, $conf, $node) = @_;
2620
2621     foreach_drive($conf, sub {
2622         my ($ds, $drive) = @_;
2623
2624         my $volid = $drive->{file};
2625         return if !$volid;
2626
2627         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2628         return if !$sid;
2629
2630         # check if storage is available on both nodes
2631         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2632         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2633    });
2634 }
2635
2636 # list nodes where all VM images are available (used by has_feature API)
2637 sub shared_nodes {
2638     my ($conf, $storecfg) = @_;
2639
2640     my $nodelist = PVE::Cluster::get_nodelist();
2641     my $nodehash = { map { $_ => 1 } @$nodelist };
2642     my $nodename = PVE::INotify::nodename();
2643
2644     foreach_drive($conf, sub {
2645         my ($ds, $drive) = @_;
2646
2647         my $volid = $drive->{file};
2648         return if !$volid;
2649
2650         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2651         if ($storeid) {
2652             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2653             if ($scfg->{disable}) {
2654                 $nodehash = {};
2655             } elsif (my $avail = $scfg->{nodes}) {
2656                 foreach my $node (keys %$nodehash) {
2657                     delete $nodehash->{$node} if !$avail->{$node};
2658                 }
2659             } elsif (!$scfg->{shared}) {
2660                 foreach my $node (keys %$nodehash) {
2661                     delete $nodehash->{$node} if $node ne $nodename
2662                 }
2663             }
2664         }
2665     });
2666
2667     return $nodehash
2668 }
2669
2670 sub check_cmdline {
2671     my ($pidfile, $pid) = @_;
2672
2673     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2674     if (defined($fh)) {
2675         my $line = <$fh>;
2676         $fh->close;
2677         return undef if !$line;
2678         my @param = split(/\0/, $line);
2679
2680         my $cmd = $param[0];
2681         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2682
2683         for (my $i = 0; $i < scalar (@param); $i++) {
2684             my $p = $param[$i];
2685             next if !$p;
2686             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2687                 my $p = $param[$i+1];
2688                 return 1 if $p && ($p eq $pidfile);
2689                 return undef;
2690             }
2691         }
2692     }
2693     return undef;
2694 }
2695
2696 sub check_running {
2697     my ($vmid, $nocheck, $node) = @_;
2698
2699     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2700
2701     die "unable to find configuration file for VM $vmid - no such machine\n"
2702         if !$nocheck && ! -f $filename;
2703
2704     my $pidfile = pidfile_name($vmid);
2705
2706     if (my $fd = IO::File->new("<$pidfile")) {
2707         my $st = stat($fd);
2708         my $line = <$fd>;
2709         close($fd);
2710
2711         my $mtime = $st->mtime;
2712         if ($mtime > time()) {
2713             warn "file '$filename' modified in future\n";
2714         }
2715
2716         if ($line =~ m/^(\d+)$/) {
2717             my $pid = $1;
2718             if (check_cmdline($pidfile, $pid)) {
2719                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2720                     return $pid;
2721                 }
2722             }
2723         }
2724     }
2725
2726     return undef;
2727 }
2728
2729 sub vzlist {
2730
2731     my $vzlist = config_list();
2732
2733     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2734
2735     while (defined(my $de = $fd->read)) {
2736         next if $de !~ m/^(\d+)\.pid$/;
2737         my $vmid = $1;
2738         next if !defined($vzlist->{$vmid});
2739         if (my $pid = check_running($vmid)) {
2740             $vzlist->{$vmid}->{pid} = $pid;
2741         }
2742     }
2743
2744     return $vzlist;
2745 }
2746
2747 sub disksize {
2748     my ($storecfg, $conf) = @_;
2749
2750     my $bootdisk = $conf->{bootdisk};
2751     return undef if !$bootdisk;
2752     return undef if !is_valid_drivename($bootdisk);
2753
2754     return undef if !$conf->{$bootdisk};
2755
2756     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2757     return undef if !defined($drive);
2758
2759     return undef if drive_is_cdrom($drive);
2760
2761     my $volid = $drive->{file};
2762     return undef if !$volid;
2763
2764     return $drive->{size};
2765 }
2766
2767 my $last_proc_pid_stat;
2768
2769 # get VM status information
2770 # This must be fast and should not block ($full == false)
2771 # We only query KVM using QMP if $full == true (this can be slow)
2772 sub vmstatus {
2773     my ($opt_vmid, $full) = @_;
2774
2775     my $res = {};
2776
2777     my $storecfg = PVE::Storage::config();
2778
2779     my $list = vzlist();
2780     my $defaults = load_defaults();
2781
2782     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2783
2784     my $cpucount = $cpuinfo->{cpus} || 1;
2785
2786     foreach my $vmid (keys %$list) {
2787         next if $opt_vmid && ($vmid ne $opt_vmid);
2788
2789         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2790         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2791
2792         my $d = {};
2793         $d->{pid} = $list->{$vmid}->{pid};
2794
2795         # fixme: better status?
2796         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2797
2798         my $size = disksize($storecfg, $conf);
2799         if (defined($size)) {
2800             $d->{disk} = 0; # no info available
2801             $d->{maxdisk} = $size;
2802         } else {
2803             $d->{disk} = 0;
2804             $d->{maxdisk} = 0;
2805         }
2806
2807         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2808             * ($conf->{cores} || $defaults->{cores});
2809         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2810         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2811
2812         $d->{name} = $conf->{name} || "VM $vmid";
2813         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2814             : $defaults->{memory}*(1024*1024);
2815
2816         if ($conf->{balloon}) {
2817             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2818             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2819                 : $defaults->{shares};
2820         }
2821
2822         $d->{uptime} = 0;
2823         $d->{cpu} = 0;
2824         $d->{mem} = 0;
2825
2826         $d->{netout} = 0;
2827         $d->{netin} = 0;
2828
2829         $d->{diskread} = 0;
2830         $d->{diskwrite} = 0;
2831
2832         $d->{template} = PVE::QemuConfig->is_template($conf);
2833
2834         $d->{serial} = 1 if conf_has_serial($conf);
2835
2836         $res->{$vmid} = $d;
2837     }
2838
2839     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2840     foreach my $dev (keys %$netdev) {
2841         next if $dev !~ m/^tap([1-9]\d*)i/;
2842         my $vmid = $1;
2843         my $d = $res->{$vmid};
2844         next if !$d;
2845
2846         $d->{netout} += $netdev->{$dev}->{receive};
2847         $d->{netin} += $netdev->{$dev}->{transmit};
2848
2849         if ($full) {
2850             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2851             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2852         }
2853
2854     }
2855
2856     my $ctime = gettimeofday;
2857
2858     foreach my $vmid (keys %$list) {
2859
2860         my $d = $res->{$vmid};
2861         my $pid = $d->{pid};
2862         next if !$pid;
2863
2864         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2865         next if !$pstat; # not running
2866
2867         my $used = $pstat->{utime} + $pstat->{stime};
2868
2869         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2870
2871         if ($pstat->{vsize}) {
2872             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2873         }
2874
2875         my $old = $last_proc_pid_stat->{$pid};
2876         if (!$old) {
2877             $last_proc_pid_stat->{$pid} = {
2878                 time => $ctime,
2879                 used => $used,
2880                 cpu => 0,
2881             };
2882             next;
2883         }
2884
2885         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2886
2887         if ($dtime > 1000) {
2888             my $dutime = $used -  $old->{used};
2889
2890             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => $d->{cpu},
2895             };
2896         } else {
2897             $d->{cpu} = $old->{cpu};
2898         }
2899     }
2900
2901     return $res if !$full;
2902
2903     my $qmpclient = PVE::QMPClient->new();
2904
2905     my $ballooncb = sub {
2906         my ($vmid, $resp) = @_;
2907
2908         my $info = $resp->{'return'};
2909         return if !$info->{max_mem};
2910
2911         my $d = $res->{$vmid};
2912
2913         # use memory assigned to VM
2914         $d->{maxmem} = $info->{max_mem};
2915         $d->{balloon} = $info->{actual};
2916
2917         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2918             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2919             $d->{freemem} = $info->{free_mem};
2920         }
2921
2922         $d->{ballooninfo} = $info;
2923     };
2924
2925     my $blockstatscb = sub {
2926         my ($vmid, $resp) = @_;
2927         my $data = $resp->{'return'} || [];
2928         my $totalrdbytes = 0;
2929         my $totalwrbytes = 0;
2930
2931         for my $blockstat (@$data) {
2932             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2933             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2934
2935             $blockstat->{device} =~ s/drive-//;
2936             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2937         }
2938         $res->{$vmid}->{diskread} = $totalrdbytes;
2939         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2940     };
2941
2942     my $statuscb = sub {
2943         my ($vmid, $resp) = @_;
2944
2945         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2946         # this fails if ballon driver is not loaded, so this must be
2947         # the last commnand (following command are aborted if this fails).
2948         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2949
2950         my $status = 'unknown';
2951         if (!defined($status = $resp->{'return'}->{status})) {
2952             warn "unable to get VM status\n";
2953             return;
2954         }
2955
2956         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2957     };
2958
2959     foreach my $vmid (keys %$list) {
2960         next if $opt_vmid && ($vmid ne $opt_vmid);
2961         next if !$res->{$vmid}->{pid}; # not running
2962         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2963     }
2964
2965     $qmpclient->queue_execute(undef, 2);
2966
2967     foreach my $vmid (keys %$list) {
2968         next if $opt_vmid && ($vmid ne $opt_vmid);
2969         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2970     }
2971
2972     return $res;
2973 }
2974
2975 sub foreach_drive {
2976     my ($conf, $func, @param) = @_;
2977
2978     foreach my $ds (valid_drive_names()) {
2979         next if !defined($conf->{$ds});
2980
2981         my $drive = parse_drive($ds, $conf->{$ds});
2982         next if !$drive;
2983
2984         &$func($ds, $drive, @param);
2985     }
2986 }
2987
2988 sub foreach_volid {
2989     my ($conf, $func, @param) = @_;
2990
2991     my $volhash = {};
2992
2993     my $test_volid = sub {
2994         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2995
2996         return if !$volid;
2997
2998         $volhash->{$volid}->{cdrom} //= 1;
2999         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3000
3001         $volhash->{$volid}->{replicate} //= 0;
3002         $volhash->{$volid}->{replicate} = 1 if $replicate;
3003
3004         $volhash->{$volid}->{shared} //= 0;
3005         $volhash->{$volid}->{shared} = 1 if $shared;
3006
3007         $volhash->{$volid}->{referenced_in_config} //= 0;
3008         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3009
3010         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3011             if defined($snapname);
3012     };
3013
3014     foreach_drive($conf, sub {
3015         my ($ds, $drive) = @_;
3016         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3017     });
3018
3019     foreach my $snapname (keys %{$conf->{snapshots}}) {
3020         my $snap = $conf->{snapshots}->{$snapname};
3021         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3022         foreach_drive($snap, sub {
3023             my ($ds, $drive) = @_;
3024             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3025         });
3026     }
3027
3028     foreach my $volid (keys %$volhash) {
3029         &$func($volid, $volhash->{$volid}, @param);
3030     }
3031 }
3032
3033 sub conf_has_serial {
3034     my ($conf) = @_;
3035
3036     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3037         if ($conf->{"serial$i"}) {
3038             return 1;
3039         }
3040     }
3041
3042     return 0;
3043 }
3044
3045 sub vga_conf_has_spice {
3046     my ($vga) = @_;
3047
3048     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3049
3050     return $1 || 1;
3051 }
3052
3053 sub config_to_command {
3054     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3055
3056     my $cmd = [];
3057     my $globalFlags = [];
3058     my $machineFlags = [];
3059     my $rtcFlags = [];
3060     my $cpuFlags = [];
3061     my $devices = [];
3062     my $pciaddr = '';
3063     my $bridges = {};
3064     my $kvmver = kvm_user_version();
3065     my $vernum = 0; # unknown
3066     my $ostype = $conf->{ostype};
3067     my $winversion = windows_version($ostype);
3068     my $kvm = $conf->{kvm} // 1;
3069
3070     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3071
3072     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3073         $vernum = $1*1000000+$2*1000;
3074     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3075         $vernum = $1*1000000+$2*1000+$3;
3076     }
3077
3078     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3079
3080     my $have_ovz = -f '/proc/vz/vestat';
3081
3082     my $q35 = machine_type_is_q35($conf);
3083     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3084     my $machine_type = $forcemachine || $conf->{machine};
3085     my $use_old_bios_files = undef;
3086     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3087
3088     my $cpuunits = defined($conf->{cpuunits}) ?
3089             $conf->{cpuunits} : $defaults->{cpuunits};
3090
3091     push @$cmd, '/usr/bin/kvm';
3092
3093     push @$cmd, '-id', $vmid;
3094
3095     my $vmname = $conf->{name} || "vm$vmid";
3096
3097     push @$cmd, '-name', $vmname;
3098
3099     my $use_virtio = 0;
3100
3101     my $qmpsocket = qmp_socket($vmid);
3102     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3103     push @$cmd, '-mon', "chardev=qmp,mode=control";
3104
3105
3106     push @$cmd, '-pidfile' , pidfile_name($vmid);
3107
3108     push @$cmd, '-daemonize';
3109
3110     if ($conf->{smbios1}) {
3111         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3112     }
3113
3114     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3115         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3116
3117         my $path;
3118         my $format;
3119         if (my $efidisk = $conf->{efidisk0}) {
3120             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3121             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3122             $format = $d->{format};
3123             if ($storeid) {
3124                 $path = PVE::Storage::path($storecfg, $d->{file});
3125                 if (!defined($format)) {
3126                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3127                     $format = qemu_img_format($scfg, $volname);
3128                 }
3129             } else {
3130                 $path = $d->{file};
3131                 die "efidisk format must be specified\n"
3132                     if !defined($format);
3133             }
3134         } else {
3135             warn "no efidisk configured! Using temporary efivars disk.\n";
3136             $path = "/tmp/$vmid-ovmf.fd";
3137             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3138             $format = 'raw';
3139         }
3140
3141         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3142         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3143     }
3144
3145
3146     # add usb controllers
3147     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3148     push @$devices, @usbcontrollers if @usbcontrollers;
3149     my $vga = $conf->{vga};
3150
3151     my $qxlnum = vga_conf_has_spice($vga);
3152     $vga = 'qxl' if $qxlnum;
3153
3154     if (!$vga) {
3155         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3156             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3157         } else {
3158             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3159         }
3160     }
3161
3162     # enable absolute mouse coordinates (needed by vnc)
3163     my $tablet;
3164     if (defined($conf->{tablet})) {
3165         $tablet = $conf->{tablet};
3166     } else {
3167         $tablet = $defaults->{tablet};
3168         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3169         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3170     }
3171
3172     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3173
3174     my $kvm_off = 0;
3175     my $gpu_passthrough;
3176
3177     # host pci devices
3178     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3179         my $d = parse_hostpci($conf->{"hostpci$i"});
3180         next if !$d;
3181
3182         my $pcie = $d->{pcie};
3183         if($pcie){
3184             die "q35 machine model is not enabled" if !$q35;
3185             $pciaddr = print_pcie_addr("hostpci$i");
3186         }else{
3187             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3188         }
3189
3190         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3191         my $romfile = $d->{romfile};
3192
3193         my $xvga = '';
3194         if ($d->{'x-vga'}) {
3195             $xvga = ',x-vga=on';
3196             $kvm_off = 1;
3197             $vga = 'none';
3198             $gpu_passthrough = 1;
3199
3200             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3201                 $xvga = "";
3202             }
3203         }
3204         my $pcidevices = $d->{pciid};
3205         my $multifunction = 1 if @$pcidevices > 1;
3206
3207         my $j=0;
3208         foreach my $pcidevice (@$pcidevices) {
3209
3210             my $id = "hostpci$i";
3211             $id .= ".$j" if $multifunction;
3212             my $addr = $pciaddr;
3213             $addr .= ".$j" if $multifunction;
3214             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3215
3216             if($j == 0){
3217                 $devicestr .= "$rombar$xvga";
3218                 $devicestr .= ",multifunction=on" if $multifunction;
3219                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3220             }
3221
3222             push @$devices, '-device', $devicestr;
3223             $j++;
3224         }
3225     }
3226
3227     # usb devices
3228     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3229     push @$devices, @usbdevices if @usbdevices;
3230     # serial devices
3231     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3232         if (my $path = $conf->{"serial$i"}) {
3233             if ($path eq 'socket') {
3234                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3235                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3236                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3237             } else {
3238                 die "no such serial device\n" if ! -c $path;
3239                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3240                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3241             }
3242         }
3243     }
3244
3245     # parallel devices
3246     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3247         if (my $path = $conf->{"parallel$i"}) {
3248             die "no such parallel device\n" if ! -c $path;
3249             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3250             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3251             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3252         }
3253     }
3254
3255
3256     my $sockets = 1;
3257     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3258     $sockets = $conf->{sockets} if  $conf->{sockets};
3259
3260     my $cores = $conf->{cores} || 1;
3261
3262     my $maxcpus = $sockets * $cores;
3263
3264     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3265
3266     my $allowed_vcpus = $cpuinfo->{cpus};
3267
3268     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3269         if ($allowed_vcpus < $maxcpus);
3270
3271     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3272
3273         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3274         for (my $i = 2; $i <= $vcpus; $i++)  {
3275             my $cpustr = print_cpu_device($conf,$i);
3276             push @$cmd, '-device', $cpustr;
3277         }
3278
3279     } else {
3280
3281         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3282     }
3283     push @$cmd, '-nodefaults';
3284
3285     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3286
3287     my $bootindex_hash = {};
3288     my $i = 1;
3289     foreach my $o (split(//, $bootorder)) {
3290         $bootindex_hash->{$o} = $i*100;
3291         $i++;
3292     }
3293
3294     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3301
3302     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3303         my $socket = vnc_socket($vmid);
3304         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3305     } else {
3306         push @$cmd, '-nographic';
3307     }
3308
3309     # time drift fix
3310     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3311
3312     my $useLocaltime = $conf->{localtime};
3313
3314     if ($winversion >= 5) { # windows
3315         $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317         # use time drift fix when acpi is enabled
3318         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319             $tdf = 1 if !defined($conf->{tdf});
3320         }
3321     }
3322
3323     if ($winversion >= 6) {
3324         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325         push @$cmd, '-no-hpet';
3326     }
3327
3328     push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330     if (!$kvm) {
3331         push @$machineFlags, 'accel=tcg';
3332     }
3333
3334     if ($machine_type) {
3335         push @$machineFlags, "type=${machine_type}";
3336     }
3337
3338     if ($conf->{startdate}) {
3339         push @$rtcFlags, "base=$conf->{startdate}";
3340     } elsif ($useLocaltime) {
3341         push @$rtcFlags, 'base=localtime';
3342     }
3343
3344     my $cpu = $kvm ? "kvm64" : "qemu64";
3345     if (my $cputype = $conf->{cpu}) {
3346         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3347             or die "Cannot parse cpu description: $cputype\n";
3348         $cpu = $cpuconf->{cputype};
3349         $kvm_off = 1 if $cpuconf->{hidden};
3350
3351         if (defined(my $flags = $cpuconf->{flags})) {
3352             push @$cpuFlags, split(";", $flags);
3353         }
3354     }
3355
3356     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3357
3358     push @$cpuFlags , '-x2apic'
3359         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3360
3361     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3362
3363     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3364
3365     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3366
3367         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3368         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3369     }
3370
3371     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3372
3373     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3374
3375     push @$cpuFlags, 'kvm=off' if $kvm_off;
3376
3377     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3378         die "internal error"; # should not happen
3379
3380     push @$cpuFlags, "vendor=${cpu_vendor}"
3381         if $cpu_vendor ne 'default';
3382
3383     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3384
3385     push @$cmd, '-cpu', $cpu;
3386
3387     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3388
3389     push @$cmd, '-S' if $conf->{freeze};
3390
3391     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3392
3393     # enable sound
3394     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3395     #push @$cmd, '-soundhw', 'es1370';
3396     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3397
3398     if($conf->{agent}) {
3399         my $qgasocket = qmp_socket($vmid, 1);
3400         my $pciaddr = print_pci_addr("qga0", $bridges);
3401         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3402         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3403         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3404     }
3405
3406     my $spice_port;
3407
3408     if ($qxlnum) {
3409         if ($qxlnum > 1) {
3410             if ($winversion){
3411                 for(my $i = 1; $i < $qxlnum; $i++){
3412                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3413                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3414                 }
3415             } else {
3416                 # assume other OS works like Linux
3417                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3418                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3419             }
3420         }
3421
3422         my $pciaddr = print_pci_addr("spice", $bridges);
3423
3424         my $nodename = PVE::INotify::nodename();
3425         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3426         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3427         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3428         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3429         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3430
3431         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3432
3433         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3434         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3435         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3436     }
3437
3438     # enable balloon by default, unless explicitly disabled
3439     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3440         $pciaddr = print_pci_addr("balloon0", $bridges);
3441         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3442     }
3443
3444     if ($conf->{watchdog}) {
3445         my $wdopts = parse_watchdog($conf->{watchdog});
3446         $pciaddr = print_pci_addr("watchdog", $bridges);
3447         my $watchdog = $wdopts->{model} || 'i6300esb';
3448         push @$devices, '-device', "$watchdog$pciaddr";
3449         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3450     }
3451
3452     my $vollist = [];
3453     my $scsicontroller = {};
3454     my $ahcicontroller = {};
3455     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3456
3457     # Add iscsi initiator name if available
3458     if (my $initiator = get_initiator_name()) {
3459         push @$devices, '-iscsi', "initiator-name=$initiator";
3460     }
3461
3462     foreach_drive($conf, sub {
3463         my ($ds, $drive) = @_;
3464
3465         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3466             push @$vollist, $drive->{file};
3467         }
3468
3469         # ignore efidisk here, already added in bios/fw handling code above
3470         return if $drive->{interface} eq 'efidisk';
3471
3472         $use_virtio = 1 if $ds =~ m/^virtio/;
3473
3474         if (drive_is_cdrom ($drive)) {
3475             if ($bootindex_hash->{d}) {
3476                 $drive->{bootindex} = $bootindex_hash->{d};
3477                 $bootindex_hash->{d} += 1;
3478             }
3479         } else {
3480             if ($bootindex_hash->{c}) {
3481                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3482                 $bootindex_hash->{c} += 1;
3483             }
3484         }
3485
3486         if($drive->{interface} eq 'virtio'){
3487            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3488         }
3489
3490         if ($drive->{interface} eq 'scsi') {
3491
3492             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3493
3494             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3495             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3496
3497             my $iothread = '';
3498             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3499                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3500                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3501             } elsif ($drive->{iothread}) {
3502                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3503             }
3504
3505             my $queues = '';
3506             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3507                 $queues = ",num_queues=$drive->{queues}";
3508             }
3509
3510             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3511             $scsicontroller->{$controller}=1;
3512         }
3513
3514         if ($drive->{interface} eq 'sata') {
3515            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3516            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3517            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3518            $ahcicontroller->{$controller}=1;
3519         }
3520
3521         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3522         push @$devices, '-drive',$drive_cmd;
3523         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3524     });
3525
3526     for (my $i = 0; $i < $MAX_NETS; $i++) {
3527          next if !$conf->{"net$i"};
3528          my $d = parse_net($conf->{"net$i"});
3529          next if !$d;
3530
3531          $use_virtio = 1 if $d->{model} eq 'virtio';
3532
3533          if ($bootindex_hash->{n}) {
3534             $d->{bootindex} = $bootindex_hash->{n};
3535             $bootindex_hash->{n} += 1;
3536          }
3537
3538          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3539          push @$devices, '-netdev', $netdevfull;
3540
3541          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3542          push @$devices, '-device', $netdevicefull;
3543     }
3544
3545     if (!$q35) {
3546         # add pci bridges
3547         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3548            $bridges->{1} = 1;
3549            $bridges->{2} = 1;
3550         }
3551
3552         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3553
3554         while (my ($k, $v) = each %$bridges) {
3555             $pciaddr = print_pci_addr("pci.$k");
3556             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3557         }
3558     }
3559
3560     # add custom args
3561     if ($conf->{args}) {
3562         my $aa = PVE::Tools::split_args($conf->{args});
3563         push @$cmd, @$aa;
3564     }
3565
3566     push @$cmd, @$devices;
3567     push @$cmd, '-rtc', join(',', @$rtcFlags)
3568         if scalar(@$rtcFlags);
3569     push @$cmd, '-machine', join(',', @$machineFlags)
3570         if scalar(@$machineFlags);
3571     push @$cmd, '-global', join(',', @$globalFlags)
3572         if scalar(@$globalFlags);
3573
3574     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3575 }
3576
3577 sub vnc_socket {
3578     my ($vmid) = @_;
3579     return "${var_run_tmpdir}/$vmid.vnc";
3580 }
3581
3582 sub spice_port {
3583     my ($vmid) = @_;
3584
3585     my $res = vm_mon_cmd($vmid, 'query-spice');
3586
3587     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3588 }
3589
3590 sub qmp_socket {
3591     my ($vmid, $qga) = @_;
3592     my $sockettype = $qga ? 'qga' : 'qmp';
3593     return "${var_run_tmpdir}/$vmid.$sockettype";
3594 }
3595
3596 sub pidfile_name {
3597     my ($vmid) = @_;
3598     return "${var_run_tmpdir}/$vmid.pid";
3599 }
3600
3601 sub vm_devices_list {
3602     my ($vmid) = @_;
3603
3604     my $res = vm_mon_cmd($vmid, 'query-pci');
3605     my $devices_to_check = [];
3606     my $devices = {};
3607     foreach my $pcibus (@$res) {
3608         push @$devices_to_check, @{$pcibus->{devices}},
3609     }
3610
3611     while (@$devices_to_check) {
3612         my $to_check = [];
3613         for my $d (@$devices_to_check) {
3614             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3615             next if !$d->{'pci_bridge'};
3616
3617             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3618             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3619         }
3620         $devices_to_check = $to_check;
3621     }
3622
3623     my $resblock = vm_mon_cmd($vmid, 'query-block');
3624     foreach my $block (@$resblock) {
3625         if($block->{device} =~ m/^drive-(\S+)/){
3626                 $devices->{$1} = 1;
3627         }
3628     }
3629
3630     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3631     foreach my $mice (@$resmice) {
3632         if ($mice->{name} eq 'QEMU HID Tablet') {
3633             $devices->{tablet} = 1;
3634             last;
3635         }
3636     }
3637
3638     # for usb devices there is no query-usb
3639     # but we can iterate over the entries in
3640     # qom-list path=/machine/peripheral
3641     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3642     foreach my $per (@$resperipheral) {
3643         if ($per->{name} =~ m/^usb\d+$/) {
3644             $devices->{$per->{name}} = 1;
3645         }
3646     }
3647
3648     return $devices;
3649 }
3650
3651 sub vm_deviceplug {
3652     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3653
3654     my $q35 = machine_type_is_q35($conf);
3655
3656     my $devices_list = vm_devices_list($vmid);
3657     return 1 if defined($devices_list->{$deviceid});
3658
3659     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3660
3661     if ($deviceid eq 'tablet') {
3662
3663         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3664
3665     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3666
3667         die "usb hotplug currently not reliable\n";
3668         # since we can't reliably hot unplug all added usb devices
3669         # and usb passthrough disables live migration
3670         # we disable usb hotplugging for now
3671         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3672
3673     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3674
3675         qemu_iothread_add($vmid, $deviceid, $device);
3676
3677         qemu_driveadd($storecfg, $vmid, $device);
3678         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3679
3680         qemu_deviceadd($vmid, $devicefull);
3681         eval { qemu_deviceaddverify($vmid, $deviceid); };
3682         if (my $err = $@) {
3683             eval { qemu_drivedel($vmid, $deviceid); };
3684             warn $@ if $@;
3685             die $err;
3686         }
3687
3688     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3689
3690
3691         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3692         my $pciaddr = print_pci_addr($deviceid);
3693         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3694
3695         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3696
3697         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3698             qemu_iothread_add($vmid, $deviceid, $device);
3699             $devicefull .= ",iothread=iothread-$deviceid";
3700         }
3701
3702         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3703             $devicefull .= ",num_queues=$device->{queues}";
3704         }
3705
3706         qemu_deviceadd($vmid, $devicefull);
3707         qemu_deviceaddverify($vmid, $deviceid);
3708
3709     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3710
3711         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3712         qemu_driveadd($storecfg, $vmid, $device);
3713
3714         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3715         eval { qemu_deviceadd($vmid, $devicefull); };
3716         if (my $err = $@) {
3717             eval { qemu_drivedel($vmid, $deviceid); };
3718             warn $@ if $@;
3719             die $err;
3720         }
3721
3722     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3723
3724         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3725
3726         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3727         my $use_old_bios_files = undef;
3728         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3729
3730         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3731         qemu_deviceadd($vmid, $netdevicefull);
3732         eval { qemu_deviceaddverify($vmid, $deviceid); };
3733         if (my $err = $@) {
3734             eval { qemu_netdevdel($vmid, $deviceid); };
3735             warn $@ if $@;
3736             die $err;
3737         }
3738
3739     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3740
3741         my $bridgeid = $2;
3742         my $pciaddr = print_pci_addr($deviceid);
3743         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3744
3745         qemu_deviceadd($vmid, $devicefull);
3746         qemu_deviceaddverify($vmid, $deviceid);
3747
3748     } else {
3749         die "can't hotplug device '$deviceid'\n";
3750     }
3751
3752     return 1;
3753 }
3754
3755 # fixme: this should raise exceptions on error!
3756 sub vm_deviceunplug {
3757     my ($vmid, $conf, $deviceid) = @_;
3758
3759     my $devices_list = vm_devices_list($vmid);
3760     return 1 if !defined($devices_list->{$deviceid});
3761
3762     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3763
3764     if ($deviceid eq 'tablet') {
3765
3766         qemu_devicedel($vmid, $deviceid);
3767
3768     } elsif ($deviceid =~ m/^usb\d+$/) {
3769
3770         die "usb hotplug currently not reliable\n";
3771         # when unplugging usb devices this way,
3772         # there may be remaining usb controllers/hubs
3773         # so we disable it for now
3774         qemu_devicedel($vmid, $deviceid);
3775         qemu_devicedelverify($vmid, $deviceid);
3776
3777     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3778
3779         qemu_devicedel($vmid, $deviceid);
3780         qemu_devicedelverify($vmid, $deviceid);
3781         qemu_drivedel($vmid, $deviceid);
3782         qemu_iothread_del($conf, $vmid, $deviceid);
3783
3784     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3785
3786         qemu_devicedel($vmid, $deviceid);
3787         qemu_devicedelverify($vmid, $deviceid);
3788         qemu_iothread_del($conf, $vmid, $deviceid);
3789
3790     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3791
3792         qemu_devicedel($vmid, $deviceid);
3793         qemu_drivedel($vmid, $deviceid);
3794         qemu_deletescsihw($conf, $vmid, $deviceid);
3795
3796     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3797
3798         qemu_devicedel($vmid, $deviceid);
3799         qemu_devicedelverify($vmid, $deviceid);
3800         qemu_netdevdel($vmid, $deviceid);
3801
3802     } else {
3803         die "can't unplug device '$deviceid'\n";
3804     }
3805
3806     return 1;
3807 }
3808
3809 sub qemu_deviceadd {
3810     my ($vmid, $devicefull) = @_;
3811
3812     $devicefull = "driver=".$devicefull;
3813     my %options =  split(/[=,]/, $devicefull);
3814
3815     vm_mon_cmd($vmid, "device_add" , %options);
3816 }
3817
3818 sub qemu_devicedel {
3819     my ($vmid, $deviceid) = @_;
3820
3821     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3822 }
3823
3824 sub qemu_iothread_add {
3825     my($vmid, $deviceid, $device) = @_;
3826
3827     if ($device->{iothread}) {
3828         my $iothreads = vm_iothreads_list($vmid);
3829         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3830     }
3831 }
3832
3833 sub qemu_iothread_del {
3834     my($conf, $vmid, $deviceid) = @_;
3835
3836     my $device = parse_drive($deviceid, $conf->{$deviceid});
3837     if ($device->{iothread}) {
3838         my $iothreads = vm_iothreads_list($vmid);
3839         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3840     }
3841 }
3842
3843 sub qemu_objectadd {
3844     my($vmid, $objectid, $qomtype) = @_;
3845
3846     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3847
3848     return 1;
3849 }
3850
3851 sub qemu_objectdel {
3852     my($vmid, $objectid) = @_;
3853
3854     vm_mon_cmd($vmid, "object-del", id => $objectid);
3855
3856     return 1;
3857 }
3858
3859 sub qemu_driveadd {
3860     my ($storecfg, $vmid, $device) = @_;
3861
3862     my $drive = print_drive_full($storecfg, $vmid, $device);
3863     $drive =~ s/\\/\\\\/g;
3864     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3865
3866     # If the command succeeds qemu prints: "OK"
3867     return 1 if $ret =~ m/OK/s;
3868
3869     die "adding drive failed: $ret\n";
3870 }
3871
3872 sub qemu_drivedel {
3873     my($vmid, $deviceid) = @_;
3874
3875     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3876     $ret =~ s/^\s+//;
3877
3878     return 1 if $ret eq "";
3879
3880     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3881     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3882
3883     die "deleting drive $deviceid failed : $ret\n";
3884 }
3885
3886 sub qemu_deviceaddverify {
3887     my ($vmid, $deviceid) = @_;
3888
3889     for (my $i = 0; $i <= 5; $i++) {
3890          my $devices_list = vm_devices_list($vmid);
3891          return 1 if defined($devices_list->{$deviceid});
3892          sleep 1;
3893     }
3894
3895     die "error on hotplug device '$deviceid'\n";
3896 }
3897
3898
3899 sub qemu_devicedelverify {
3900     my ($vmid, $deviceid) = @_;
3901
3902     # need to verify that the device is correctly removed as device_del
3903     # is async and empty return is not reliable
3904
3905     for (my $i = 0; $i <= 5; $i++) {
3906          my $devices_list = vm_devices_list($vmid);
3907          return 1 if !defined($devices_list->{$deviceid});
3908          sleep 1;
3909     }
3910
3911     die "error on hot-unplugging device '$deviceid'\n";
3912 }
3913
3914 sub qemu_findorcreatescsihw {
3915     my ($storecfg, $conf, $vmid, $device) = @_;
3916
3917     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3918
3919     my $scsihwid="$controller_prefix$controller";
3920     my $devices_list = vm_devices_list($vmid);
3921
3922     if(!defined($devices_list->{$scsihwid})) {
3923         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3924     }
3925
3926     return 1;
3927 }
3928
3929 sub qemu_deletescsihw {
3930     my ($conf, $vmid, $opt) = @_;
3931
3932     my $device = parse_drive($opt, $conf->{$opt});
3933
3934     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3935         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3936         return 1;
3937     }
3938
3939     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3940
3941     my $devices_list = vm_devices_list($vmid);
3942     foreach my $opt (keys %{$devices_list}) {
3943         if (PVE::QemuServer::is_valid_drivename($opt)) {
3944             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3945             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3946                 return 1;
3947             }
3948         }
3949     }
3950
3951     my $scsihwid="scsihw$controller";
3952
3953     vm_deviceunplug($vmid, $conf, $scsihwid);
3954
3955     return 1;
3956 }
3957
3958 sub qemu_add_pci_bridge {
3959     my ($storecfg, $conf, $vmid, $device) = @_;
3960
3961     my $bridges = {};
3962
3963     my $bridgeid;
3964
3965     print_pci_addr($device, $bridges);
3966
3967     while (my ($k, $v) = each %$bridges) {
3968         $bridgeid = $k;
3969     }
3970     return 1 if !defined($bridgeid) || $bridgeid < 1;
3971
3972     my $bridge = "pci.$bridgeid";
3973     my $devices_list = vm_devices_list($vmid);
3974
3975     if (!defined($devices_list->{$bridge})) {
3976         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3977     }
3978
3979     return 1;
3980 }
3981
3982 sub qemu_set_link_status {
3983     my ($vmid, $device, $up) = @_;
3984
3985     vm_mon_cmd($vmid, "set_link", name => $device,
3986                up => $up ? JSON::true : JSON::false);
3987 }
3988
3989 sub qemu_netdevadd {
3990     my ($vmid, $conf, $device, $deviceid) = @_;
3991
3992     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3993     my %options =  split(/[=,]/, $netdev);
3994
3995     vm_mon_cmd($vmid, "netdev_add",  %options);
3996     return 1;
3997 }
3998
3999 sub qemu_netdevdel {
4000     my ($vmid, $deviceid) = @_;
4001
4002     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4003 }
4004
4005 sub qemu_usb_hotplug {
4006     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4007
4008     return if !$device;
4009
4010     # remove the old one first
4011     vm_deviceunplug($vmid, $conf, $deviceid);
4012
4013     # check if xhci controller is necessary and available
4014     if ($device->{usb3}) {
4015
4016         my $devicelist = vm_devices_list($vmid);
4017
4018         if (!$devicelist->{xhci}) {
4019             my $pciaddr = print_pci_addr("xhci");
4020             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4021         }
4022     }
4023     my $d = parse_usb_device($device->{host});
4024     $d->{usb3} = $device->{usb3};
4025
4026     # add the new one
4027     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4028 }
4029
4030 sub qemu_cpu_hotplug {
4031     my ($vmid, $conf, $vcpus) = @_;
4032
4033     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4034
4035     my $sockets = 1;
4036     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4037     $sockets = $conf->{sockets} if  $conf->{sockets};
4038     my $cores = $conf->{cores} || 1;
4039     my $maxcpus = $sockets * $cores;
4040
4041     $vcpus = $maxcpus if !$vcpus;
4042
4043     die "you can't add more vcpus than maxcpus\n"
4044         if $vcpus > $maxcpus;
4045
4046     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4047
4048     if ($vcpus < $currentvcpus) {
4049
4050         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4051
4052             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4053                 qemu_devicedel($vmid, "cpu$i");
4054                 my $retry = 0;
4055                 my $currentrunningvcpus = undef;
4056                 while (1) {
4057                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4058                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4059                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4060                     $retry++;
4061                     sleep 1;
4062                 }
4063                 #update conf after each succesfull cpu unplug
4064                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4065                 PVE::QemuConfig->write_config($vmid, $conf);
4066             }
4067         } else {
4068             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4069         }
4070
4071         return;
4072     }
4073
4074     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4075     die "vcpus in running vm does not match its configuration\n"
4076         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4077
4078     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4079
4080         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4081             my $cpustr = print_cpu_device($conf, $i);
4082             qemu_deviceadd($vmid, $cpustr);
4083
4084             my $retry = 0;
4085             my $currentrunningvcpus = undef;
4086             while (1) {
4087                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4088                 last if scalar(@{$currentrunningvcpus}) == $i;
4089                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4090                 sleep 1;
4091                 $retry++;
4092             }
4093             #update conf after each succesfull cpu hotplug
4094             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4095             PVE::QemuConfig->write_config($vmid, $conf);
4096         }
4097     } else {
4098
4099         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4100             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4101         }
4102     }
4103 }
4104
4105 sub qemu_block_set_io_throttle {
4106     my ($vmid, $deviceid,
4107         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4108         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4109         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4110         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4111
4112     return if !check_running($vmid) ;
4113
4114     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4115         bps => int($bps),
4116         bps_rd => int($bps_rd),
4117         bps_wr => int($bps_wr),
4118         iops => int($iops),
4119         iops_rd => int($iops_rd),
4120         iops_wr => int($iops_wr),
4121         bps_max => int($bps_max),
4122         bps_rd_max => int($bps_rd_max),
4123         bps_wr_max => int($bps_wr_max),
4124         iops_max => int($iops_max),
4125         iops_rd_max => int($iops_rd_max),
4126         iops_wr_max => int($iops_wr_max),
4127         bps_max_length => int($bps_max_length),
4128         bps_rd_max_length => int($bps_rd_max_length),
4129         bps_wr_max_length => int($bps_wr_max_length),
4130         iops_max_length => int($iops_max_length),
4131         iops_rd_max_length => int($iops_rd_max_length),
4132         iops_wr_max_length => int($iops_wr_max_length),
4133     );
4134
4135 }
4136
4137 # old code, only used to shutdown old VM after update
4138 sub __read_avail {
4139     my ($fh, $timeout) = @_;
4140
4141     my $sel = new IO::Select;
4142     $sel->add($fh);
4143
4144     my $res = '';
4145     my $buf;
4146
4147     my @ready;
4148     while (scalar (@ready = $sel->can_read($timeout))) {
4149         my $count;
4150         if ($count = $fh->sysread($buf, 8192)) {
4151             if ($buf =~ /^(.*)\(qemu\) $/s) {
4152                 $res .= $1;
4153                 last;
4154             } else {
4155                 $res .= $buf;
4156             }
4157         } else {
4158             if (!defined($count)) {
4159                 die "$!\n";
4160             }
4161             last;
4162         }
4163     }
4164
4165     die "monitor read timeout\n" if !scalar(@ready);
4166
4167     return $res;
4168 }
4169
4170 sub qemu_block_resize {
4171     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4172
4173     my $running = check_running($vmid);
4174
4175     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4176
4177     return if !$running;
4178
4179     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4180
4181 }
4182
4183 sub qemu_volume_snapshot {
4184     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4185
4186     my $running = check_running($vmid);
4187
4188     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4189         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4190     } else {
4191         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4192     }
4193 }
4194
4195 sub qemu_volume_snapshot_delete {
4196     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4197
4198     my $running = check_running($vmid);
4199
4200     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4201         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4202     } else {
4203         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4204     }
4205 }
4206
4207 sub set_migration_caps {
4208     my ($vmid) = @_;
4209
4210     my $cap_ref = [];
4211
4212     my $enabled_cap = {
4213         "auto-converge" => 1,
4214         "xbzrle" => 1,
4215         "x-rdma-pin-all" => 0,
4216         "zero-blocks" => 0,
4217         "compress" => 0
4218     };
4219
4220     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4221
4222     for my $supported_capability (@$supported_capabilities) {
4223         push @$cap_ref, {
4224             capability => $supported_capability->{capability},
4225             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4226         };
4227     }
4228
4229     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4230 }
4231
4232 my $fast_plug_option = {
4233     'lock' => 1,
4234     'name' => 1,
4235     'onboot' => 1,
4236     'shares' => 1,
4237     'startup' => 1,
4238     'description' => 1,
4239     'protection' => 1,
4240     'vmstatestorage' => 1,
4241 };
4242
4243 # hotplug changes in [PENDING]
4244 # $selection hash can be used to only apply specified options, for
4245 # example: { cores => 1 } (only apply changed 'cores')
4246 # $errors ref is used to return error messages
4247 sub vmconfig_hotplug_pending {
4248     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4249
4250     my $defaults = load_defaults();
4251
4252     # commit values which do not have any impact on running VM first
4253     # Note: those option cannot raise errors, we we do not care about
4254     # $selection and always apply them.
4255
4256     my $add_error = sub {
4257         my ($opt, $msg) = @_;
4258         $errors->{$opt} = "hotplug problem - $msg";
4259     };
4260
4261     my $changes = 0;
4262     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4263         if ($fast_plug_option->{$opt}) {
4264             $conf->{$opt} = $conf->{pending}->{$opt};
4265             delete $conf->{pending}->{$opt};
4266             $changes = 1;
4267         }
4268     }
4269
4270     if ($changes) {
4271         PVE::QemuConfig->write_config($vmid, $conf);
4272         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4273     }
4274
4275     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4276
4277     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4278     while (my ($opt, $force) = each %$pending_delete_hash) {
4279         next if $selection && !$selection->{$opt};
4280         eval {
4281             if ($opt eq 'hotplug') {
4282                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4283             } elsif ($opt eq 'tablet') {
4284                 die "skip\n" if !$hotplug_features->{usb};
4285                 if ($defaults->{tablet}) {
4286                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4287                 } else {
4288                     vm_deviceunplug($vmid, $conf, $opt);
4289                 }
4290             } elsif ($opt =~ m/^usb\d+/) {
4291                 die "skip\n";
4292                 # since we cannot reliably hot unplug usb devices
4293                 # we are disabling it
4294                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4295                 vm_deviceunplug($vmid, $conf, $opt);
4296             } elsif ($opt eq 'vcpus') {
4297                 die "skip\n" if !$hotplug_features->{cpu};
4298                 qemu_cpu_hotplug($vmid, $conf, undef);
4299             } elsif ($opt eq 'balloon') {
4300                 # enable balloon device is not hotpluggable
4301                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4302                 # here we reset the ballooning value to memory
4303                 my $balloon = $conf->{memory} || $defaults->{memory};
4304                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4305             } elsif ($fast_plug_option->{$opt}) {
4306                 # do nothing
4307             } elsif ($opt =~ m/^net(\d+)$/) {
4308                 die "skip\n" if !$hotplug_features->{network};
4309                 vm_deviceunplug($vmid, $conf, $opt);
4310             } elsif (is_valid_drivename($opt)) {
4311                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4312                 vm_deviceunplug($vmid, $conf, $opt);
4313                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4314             } elsif ($opt =~ m/^memory$/) {
4315                 die "skip\n" if !$hotplug_features->{memory};
4316                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4317             } elsif ($opt eq 'cpuunits') {
4318                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4319             } elsif ($opt eq 'cpulimit') {
4320                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4321             } else {
4322                 die "skip\n";
4323             }
4324         };
4325         if (my $err = $@) {
4326             &$add_error($opt, $err) if $err ne "skip\n";
4327         } else {
4328             # save new config if hotplug was successful
4329             delete $conf->{$opt};
4330             vmconfig_undelete_pending_option($conf, $opt);
4331             PVE::QemuConfig->write_config($vmid, $conf);
4332             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4333         }
4334     }
4335
4336     my $apply_pending_cloudinit;
4337     $apply_pending_cloudinit = sub {
4338         my ($key, $value) = @_;
4339         $apply_pending_cloudinit = sub {}; # once is enough
4340
4341         my @cloudinit_opts = keys %$confdesc_cloudinit;
4342         foreach my $opt (keys %{$conf->{pending}}) {
4343             next if !grep { $_ eq $opt } @cloudinit_opts;
4344             $conf->{$opt} = delete $conf->{pending}->{$opt};
4345         }
4346
4347         my $new_conf = { %$conf };
4348         $new_conf->{$key} = $value;
4349         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4350     };
4351
4352     foreach my $opt (keys %{$conf->{pending}}) {
4353         next if $selection && !$selection->{$opt};
4354         my $value = $conf->{pending}->{$opt};
4355         eval {
4356             if ($opt eq 'hotplug') {
4357                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4358             } elsif ($opt eq 'tablet') {
4359                 die "skip\n" if !$hotplug_features->{usb};
4360                 if ($value == 1) {
4361                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4362                 } elsif ($value == 0) {
4363                     vm_deviceunplug($vmid, $conf, $opt);
4364                 }
4365             } elsif ($opt =~ m/^usb\d+$/) {
4366                 die "skip\n";
4367                 # since we cannot reliably hot unplug usb devices
4368                 # we are disabling it
4369                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4370                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4371                 die "skip\n" if !$d;
4372                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4373             } elsif ($opt eq 'vcpus') {
4374                 die "skip\n" if !$hotplug_features->{cpu};
4375                 qemu_cpu_hotplug($vmid, $conf, $value);
4376             } elsif ($opt eq 'balloon') {
4377                 # enable/disable balloning device is not hotpluggable
4378                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4379                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4380                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4381
4382                 # allow manual ballooning if shares is set to zero
4383                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4384                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4385                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4386                 }
4387             } elsif ($opt =~ m/^net(\d+)$/) {
4388                 # some changes can be done without hotplug
4389                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4390                                     $vmid, $opt, $value);
4391             } elsif (is_valid_drivename($opt)) {
4392                 # some changes can be done without hotplug
4393                 my $drive = parse_drive($opt, $value);
4394                 if (drive_is_cloudinit($drive)) {
4395                     &$apply_pending_cloudinit($opt, $value);
4396                 }
4397                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4398                                      $vmid, $opt, $value, 1);
4399             } elsif ($opt =~ m/^memory$/) { #dimms
4400                 die "skip\n" if !$hotplug_features->{memory};
4401                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4402             } elsif ($opt eq 'cpuunits') {
4403                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4404             } elsif ($opt eq 'cpulimit') {
4405                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4406                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4407             } else {
4408                 die "skip\n";  # skip non-hot-pluggable options
4409             }
4410         };
4411         if (my $err = $@) {
4412             &$add_error($opt, $err) if $err ne "skip\n";
4413         } else {
4414             # save new config if hotplug was successful
4415             $conf->{$opt} = $value;
4416             delete $conf->{pending}->{$opt};
4417             PVE::QemuConfig->write_config($vmid, $conf);
4418             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4419         }
4420     }
4421 }
4422
4423 sub try_deallocate_drive {
4424     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4425
4426     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4427         my $volid = $drive->{file};
4428         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4429             my $sid = PVE::Storage::parse_volume_id($volid);
4430             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4431
4432             # check if the disk is really unused
4433             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4434                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4435             PVE::Storage::vdisk_free($storecfg, $volid);
4436             return 1;
4437         } else {
4438             # If vm is not owner of this disk remove from config
4439             return 1;
4440         }
4441     }
4442
4443     return undef;
4444 }
4445
4446 sub vmconfig_delete_or_detach_drive {
4447     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4448
4449     my $drive = parse_drive($opt, $conf->{$opt});
4450
4451     my $rpcenv = PVE::RPCEnvironment::get();
4452     my $authuser = $rpcenv->get_user();
4453
4454     if ($force) {
4455         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4456         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4457     } else {
4458         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4459     }
4460 }
4461
4462 sub vmconfig_apply_pending {
4463     my ($vmid, $conf, $storecfg) = @_;
4464
4465     # cold plug
4466
4467     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4468     while (my ($opt, $force) = each %$pending_delete_hash) {
4469         die "internal error" if $opt =~ m/^unused/;
4470         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4471         if (!defined($conf->{$opt})) {
4472             vmconfig_undelete_pending_option($conf, $opt);
4473             PVE::QemuConfig->write_config($vmid, $conf);
4474         } elsif (is_valid_drivename($opt)) {
4475             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4476             vmconfig_undelete_pending_option($conf, $opt);
4477             delete $conf->{$opt};
4478             PVE::QemuConfig->write_config($vmid, $conf);
4479         } else {
4480             vmconfig_undelete_pending_option($conf, $opt);
4481             delete $conf->{$opt};
4482             PVE::QemuConfig->write_config($vmid, $conf);
4483         }
4484     }
4485
4486     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4487
4488     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4489         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4490
4491         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4492             # skip if nothing changed
4493         } elsif (is_valid_drivename($opt)) {
4494             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4495                 if defined($conf->{$opt});
4496             $conf->{$opt} = $conf->{pending}->{$opt};
4497         } else {
4498             $conf->{$opt} = $conf->{pending}->{$opt};
4499         }
4500
4501         delete $conf->{pending}->{$opt};
4502         PVE::QemuConfig->write_config($vmid, $conf);
4503     }
4504 }
4505
4506 my $safe_num_ne = sub {
4507     my ($a, $b) = @_;
4508
4509     return 0 if !defined($a) && !defined($b);
4510     return 1 if !defined($a);
4511     return 1 if !defined($b);
4512
4513     return $a != $b;
4514 };
4515
4516 my $safe_string_ne = sub {
4517     my ($a, $b) = @_;
4518
4519     return 0 if !defined($a) && !defined($b);
4520     return 1 if !defined($a);
4521     return 1 if !defined($b);
4522
4523     return $a ne $b;
4524 };
4525
4526 sub vmconfig_update_net {
4527     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4528
4529     my $newnet = parse_net($value);
4530
4531     if ($conf->{$opt}) {
4532         my $oldnet = parse_net($conf->{$opt});
4533
4534         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4535             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4536             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4537             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4538
4539             # for non online change, we try to hot-unplug
4540             die "skip\n" if !$hotplug;
4541             vm_deviceunplug($vmid, $conf, $opt);
4542         } else {
4543
4544             die "internal error" if $opt !~ m/net(\d+)/;
4545             my $iface = "tap${vmid}i$1";
4546
4547             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4548                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4549                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4550                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4551                 PVE::Network::tap_unplug($iface);
4552                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4553             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4554                 # Rate can be applied on its own but any change above needs to
4555                 # include the rate in tap_plug since OVS resets everything.
4556                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4557             }
4558
4559             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4560                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4561             }
4562
4563             return 1;
4564         }
4565     }
4566
4567     if ($hotplug) {
4568         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4569     } else {
4570         die "skip\n";
4571     }
4572 }
4573
4574 sub vmconfig_update_disk {
4575     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4576
4577     # fixme: do we need force?
4578
4579     my $drive = parse_drive($opt, $value);
4580
4581     if ($conf->{$opt}) {
4582
4583         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4584
4585             my $media = $drive->{media} || 'disk';
4586             my $oldmedia = $old_drive->{media} || 'disk';
4587             die "unable to change media type\n" if $media ne $oldmedia;
4588
4589             if (!drive_is_cdrom($old_drive)) {
4590
4591                 if ($drive->{file} ne $old_drive->{file}) {
4592
4593                     die "skip\n" if !$hotplug;
4594
4595                     # unplug and register as unused
4596                     vm_deviceunplug($vmid, $conf, $opt);
4597                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4598
4599                 } else {
4600                     # update existing disk
4601
4602                     # skip non hotpluggable value
4603                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4604                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4605                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4606                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4607                         die "skip\n";
4608                     }
4609
4610                     # apply throttle
4611                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4612                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4613                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4614                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4615                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4616                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4617                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4618                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4619                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4620                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4621                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4622                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4623                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4624                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4625                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4626                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4627                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4628                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4629
4630                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4631                                                    ($drive->{mbps} || 0)*1024*1024,
4632                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4633                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4634                                                    $drive->{iops} || 0,
4635                                                    $drive->{iops_rd} || 0,
4636                                                    $drive->{iops_wr} || 0,
4637                                                    ($drive->{mbps_max} || 0)*1024*1024,
4638                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4639                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4640                                                    $drive->{iops_max} || 0,
4641                                                    $drive->{iops_rd_max} || 0,
4642                                                    $drive->{iops_wr_max} || 0,
4643                                                    $drive->{bps_max_length} || 1,
4644                                                    $drive->{bps_rd_max_length} || 1,
4645                                                    $drive->{bps_wr_max_length} || 1,
4646                                                    $drive->{iops_max_length} || 1,
4647                                                    $drive->{iops_rd_max_length} || 1,
4648                                                    $drive->{iops_wr_max_length} || 1);
4649
4650                     }
4651
4652                     return 1;
4653                 }
4654
4655             } else { # cdrom
4656
4657                 if ($drive->{file} eq 'none') {
4658                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4659                     if (drive_is_cloudinit($old_drive)) {
4660                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4661                     }
4662                 } else {
4663                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4664                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4665                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4666                 }
4667
4668                 return 1;
4669             }
4670         }
4671     }
4672
4673     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4674     # hotplug new disks
4675     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4676     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4677 }
4678
4679 sub vm_start {
4680     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4681         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4682
4683     PVE::QemuConfig->lock_config($vmid, sub {
4684         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4685
4686         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4687
4688         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4689
4690         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4691
4692         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4693             vmconfig_apply_pending($vmid, $conf, $storecfg);
4694             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4695         }
4696
4697         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4698
4699         my $defaults = load_defaults();
4700
4701         # set environment variable useful inside network script
4702         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4703
4704         my $local_volumes = {};
4705
4706         if ($targetstorage) {
4707             foreach_drive($conf, sub {
4708                 my ($ds, $drive) = @_;
4709
4710                 return if drive_is_cdrom($drive);
4711
4712                 my $volid = $drive->{file};
4713
4714                 return if !$volid;
4715
4716                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4717
4718                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4719                 return if $scfg->{shared};
4720                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4721             });
4722
4723             my $format = undef;
4724
4725             foreach my $opt (sort keys %$local_volumes) {
4726
4727                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4728                 my $drive = parse_drive($opt, $conf->{$opt});
4729
4730                 #if remote storage is specified, use default format
4731                 if ($targetstorage && $targetstorage ne "1") {
4732                     $storeid = $targetstorage;
4733                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4734                     $format = $defFormat;
4735                 } else {
4736                     #else we use same format than original
4737                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4738                     $format = qemu_img_format($scfg, $volid);
4739                 }
4740
4741                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4742                 my $newdrive = $drive;
4743                 $newdrive->{format} = $format;
4744                 $newdrive->{file} = $newvolid;
4745                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4746                 $local_volumes->{$opt} = $drivestr;
4747                 #pass drive to conf for command line
4748                 $conf->{$opt} = $drivestr;
4749             }
4750         }
4751
4752         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4753
4754         my $migrate_port = 0;
4755         my $migrate_uri;
4756         if ($statefile) {
4757             if ($statefile eq 'tcp') {
4758                 my $localip = "localhost";
4759                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4760                 my $nodename = PVE::INotify::nodename();
4761
4762                 if (!defined($migration_type)) {
4763                     if (defined($datacenterconf->{migration}->{type})) {
4764                         $migration_type = $datacenterconf->{migration}->{type};
4765                     } else {
4766                         $migration_type = 'secure';
4767                     }
4768                 }
4769
4770                 if ($migration_type eq 'insecure') {
4771                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4772                     if ($migrate_network_addr) {
4773                         $localip = $migrate_network_addr;
4774                     } else {
4775                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4776                     }
4777
4778                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4779                 }
4780
4781                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4782                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4783                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4784                 push @$cmd, '-incoming', $migrate_uri;
4785                 push @$cmd, '-S';
4786
4787             } elsif ($statefile eq 'unix') {
4788                 # should be default for secure migrations as a ssh TCP forward
4789                 # tunnel is not deterministic reliable ready and fails regurarly
4790                 # to set up in time, so use UNIX socket forwards
4791                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4792                 unlink $socket_addr;
4793
4794                 $migrate_uri = "unix:$socket_addr";
4795
4796                 push @$cmd, '-incoming', $migrate_uri;
4797                 push @$cmd, '-S';
4798
4799             } else {
4800                 push @$cmd, '-loadstate', $statefile;
4801             }
4802         } elsif ($paused) {
4803             push @$cmd, '-S';
4804         }
4805
4806         # host pci devices
4807         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4808           my $d = parse_hostpci($conf->{"hostpci$i"});
4809           next if !$d;
4810           my $pcidevices = $d->{pciid};
4811           foreach my $pcidevice (@$pcidevices) {
4812                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4813
4814                 my $info = pci_device_info("0000:$pciid");
4815                 die "IOMMU not present\n" if !check_iommu_support();
4816                 die "no pci device info for device '$pciid'\n" if !$info;
4817                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4818                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4819           }
4820         }
4821
4822         PVE::Storage::activate_volumes($storecfg, $vollist);
4823
4824         if (!check_running($vmid, 1)) {
4825             eval {
4826                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4827                     outfunc => sub {}, errfunc => sub {});
4828             };
4829         }
4830
4831         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4832                                                   : $defaults->{cpuunits};
4833
4834         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4835         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4836
4837         my %properties = (
4838             Slice => 'qemu.slice',
4839             KillMode => 'none',
4840             CPUShares => $cpuunits
4841         );
4842
4843         if (my $cpulimit = $conf->{cpulimit}) {
4844             $properties{CPUQuota} = int($cpulimit * 100);
4845         }
4846         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4847
4848         my $run_qemu = sub {
4849             PVE::Tools::run_fork sub {
4850                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4851                 run_command($cmd, %run_params);
4852             };
4853         };
4854
4855         if ($conf->{hugepages}) {
4856
4857             my $code = sub {
4858                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4859                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4860
4861                 PVE::QemuServer::Memory::hugepages_mount();
4862                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4863
4864                 eval { $run_qemu->() };
4865                 if (my $err = $@) {
4866                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4867                     die $err;
4868                 }
4869
4870                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4871             };
4872             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4873
4874         } else {
4875             eval { $run_qemu->() };
4876         }
4877
4878         if (my $err = $@) {
4879             # deactivate volumes if start fails
4880             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4881             die "start failed: $err";
4882         }
4883
4884         print "migration listens on $migrate_uri\n" if $migrate_uri;
4885
4886         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4887             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4888             warn $@ if $@;
4889         }
4890
4891         #start nbd server for storage migration
4892         if ($targetstorage) {
4893             my $nodename = PVE::INotify::nodename();
4894             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4895             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4896             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4897             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4898
4899             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4900
4901             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4902
4903             foreach my $opt (sort keys %$local_volumes) {
4904                 my $volid = $local_volumes->{$opt};
4905                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4906                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4907                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4908             }
4909         }
4910
4911         if ($migratedfrom) {
4912             eval {
4913                 set_migration_caps($vmid);
4914             };
4915             warn $@ if $@;
4916
4917             if ($spice_port) {
4918                 print "spice listens on port $spice_port\n";
4919                 if ($spice_ticket) {
4920                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4921                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4922                 }
4923             }
4924
4925         } else {
4926             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4927                 if !$statefile && $conf->{balloon};
4928
4929             foreach my $opt (keys %$conf) {
4930                 next if $opt !~  m/^net\d+$/;
4931                 my $nicconf = parse_net($conf->{$opt});
4932                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4933             }
4934         }
4935
4936         vm_mon_cmd_nocheck($vmid, 'qom-set',
4937                     path => "machine/peripheral/balloon0",
4938                     property => "guest-stats-polling-interval",
4939                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4940
4941     });
4942 }
4943
4944 sub vm_mon_cmd {
4945     my ($vmid, $execute, %params) = @_;
4946
4947     my $cmd = { execute => $execute, arguments => \%params };
4948     vm_qmp_command($vmid, $cmd);
4949 }
4950
4951 sub vm_mon_cmd_nocheck {
4952     my ($vmid, $execute, %params) = @_;
4953
4954     my $cmd = { execute => $execute, arguments => \%params };
4955     vm_qmp_command($vmid, $cmd, 1);
4956 }
4957
4958 sub vm_qmp_command {
4959     my ($vmid, $cmd, $nocheck) = @_;
4960
4961     my $res;
4962
4963     my $timeout;
4964     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4965         $timeout = $cmd->{arguments}->{timeout};
4966         delete $cmd->{arguments}->{timeout};
4967     }
4968
4969     eval {
4970         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4971         my $sname = qmp_socket($vmid);
4972         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4973             my $qmpclient = PVE::QMPClient->new();
4974
4975             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4976         } else {
4977             die "unable to open monitor socket\n";
4978         }
4979     };
4980     if (my $err = $@) {
4981         syslog("err", "VM $vmid qmp command failed - $err");
4982         die $err;
4983     }
4984
4985     return $res;
4986 }
4987
4988 sub vm_human_monitor_command {
4989     my ($vmid, $cmdline) = @_;
4990
4991     my $res;
4992
4993     my $cmd = {
4994         execute => 'human-monitor-command',
4995         arguments => { 'command-line' => $cmdline},
4996     };
4997
4998     return vm_qmp_command($vmid, $cmd);
4999 }
5000
5001 sub vm_commandline {
5002     my ($storecfg, $vmid) = @_;
5003
5004     my $conf = PVE::QemuConfig->load_config($vmid);
5005
5006     my $defaults = load_defaults();
5007
5008     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5009
5010     return PVE::Tools::cmd2string($cmd);
5011 }
5012
5013 sub vm_reset {
5014     my ($vmid, $skiplock) = @_;
5015
5016     PVE::QemuConfig->lock_config($vmid, sub {
5017
5018         my $conf = PVE::QemuConfig->load_config($vmid);
5019
5020         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5021
5022         vm_mon_cmd($vmid, "system_reset");
5023     });
5024 }
5025
5026 sub get_vm_volumes {
5027     my ($conf) = @_;
5028
5029     my $vollist = [];
5030     foreach_volid($conf, sub {
5031         my ($volid, $attr) = @_;
5032
5033         return if $volid =~ m|^/|;
5034
5035         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5036         return if !$sid;
5037
5038         push @$vollist, $volid;
5039     });
5040
5041     return $vollist;
5042 }
5043
5044 sub vm_stop_cleanup {
5045     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5046
5047     eval {
5048
5049         if (!$keepActive) {
5050             my $vollist = get_vm_volumes($conf);
5051             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5052         }
5053
5054         foreach my $ext (qw(mon qmp pid vnc qga)) {
5055             unlink "/var/run/qemu-server/${vmid}.$ext";
5056         }
5057
5058         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5059     };
5060     warn $@ if $@; # avoid errors - just warn
5061 }
5062
5063 # Note: use $nockeck to skip tests if VM configuration file exists.
5064 # We need that when migration VMs to other nodes (files already moved)
5065 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5066 sub vm_stop {
5067     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5068
5069     $force = 1 if !defined($force) && !$shutdown;
5070
5071     if ($migratedfrom){
5072         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5073         kill 15, $pid if $pid;
5074         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5075         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5076         return;
5077     }
5078
5079     PVE::QemuConfig->lock_config($vmid, sub {
5080
5081         my $pid = check_running($vmid, $nocheck);
5082         return if !$pid;
5083
5084         my $conf;
5085         if (!$nocheck) {
5086             $conf = PVE::QemuConfig->load_config($vmid);
5087             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5088             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5089                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5090                 $timeout = $opts->{down} if $opts->{down};
5091             }
5092         }
5093
5094         $timeout = 60 if !defined($timeout);
5095
5096         eval {
5097             if ($shutdown) {
5098                 if (defined($conf) && $conf->{agent}) {
5099                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5100                 } else {
5101                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5102                 }
5103             } else {
5104                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5105             }
5106         };
5107         my $err = $@;
5108
5109         if (!$err) {
5110             my $count = 0;
5111             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5112                 $count++;
5113                 sleep 1;
5114             }
5115
5116             if ($count >= $timeout) {
5117                 if ($force) {
5118                     warn "VM still running - terminating now with SIGTERM\n";
5119                     kill 15, $pid;
5120                 } else {
5121                     die "VM quit/powerdown failed - got timeout\n";
5122                 }
5123             } else {
5124                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5125                 return;
5126             }
5127         } else {
5128             if ($force) {
5129                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5130                 kill 15, $pid;
5131             } else {
5132                 die "VM quit/powerdown failed\n";
5133             }
5134         }
5135
5136         # wait again
5137         $timeout = 10;
5138
5139         my $count = 0;
5140         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5141             $count++;
5142             sleep 1;
5143         }
5144
5145         if ($count >= $timeout) {
5146             warn "VM still running - terminating now with SIGKILL\n";
5147             kill 9, $pid;
5148             sleep 1;
5149         }
5150
5151         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5152    });
5153 }
5154
5155 sub vm_suspend {
5156     my ($vmid, $skiplock) = @_;
5157
5158     PVE::QemuConfig->lock_config($vmid, sub {
5159
5160         my $conf = PVE::QemuConfig->load_config($vmid);
5161
5162         PVE::QemuConfig->check_lock($conf)
5163             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5164
5165         vm_mon_cmd($vmid, "stop");
5166     });
5167 }
5168
5169 sub vm_resume {
5170     my ($vmid, $skiplock, $nocheck) = @_;
5171
5172     PVE::QemuConfig->lock_config($vmid, sub {
5173
5174         my $res = vm_mon_cmd($vmid, 'query-status');
5175         my $resume_cmd = 'cont';
5176
5177         if ($res->{status} && $res->{status} eq 'suspended') {
5178             $resume_cmd = 'system_wakeup';
5179         }
5180
5181         if (!$nocheck) {
5182
5183             my $conf = PVE::QemuConfig->load_config($vmid);
5184
5185             PVE::QemuConfig->check_lock($conf)
5186                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5187
5188             vm_mon_cmd($vmid, $resume_cmd);
5189
5190         } else {
5191             vm_mon_cmd_nocheck($vmid, $resume_cmd);
5192         }
5193     });
5194 }
5195
5196 sub vm_sendkey {
5197     my ($vmid, $skiplock, $key) = @_;
5198
5199     PVE::QemuConfig->lock_config($vmid, sub {
5200
5201         my $conf = PVE::QemuConfig->load_config($vmid);
5202
5203         # there is no qmp command, so we use the human monitor command
5204         vm_human_monitor_command($vmid, "sendkey $key");
5205     });
5206 }
5207
5208 sub vm_destroy {
5209     my ($storecfg, $vmid, $skiplock) = @_;
5210
5211     PVE::QemuConfig->lock_config($vmid, sub {
5212
5213         my $conf = PVE::QemuConfig->load_config($vmid);
5214
5215         if (!check_running($vmid)) {
5216             destroy_vm($storecfg, $vmid, undef, $skiplock);
5217         } else {
5218             die "VM $vmid is running - destroy failed\n";
5219         }
5220     });
5221 }
5222
5223 # pci helpers
5224
5225 sub file_write {
5226     my ($filename, $buf) = @_;
5227
5228     my $fh = IO::File->new($filename, "w");
5229     return undef if !$fh;
5230
5231     my $res = print $fh $buf;
5232
5233     $fh->close();
5234
5235     return $res;
5236 }
5237
5238 sub pci_device_info {
5239     my ($name) = @_;
5240
5241     my $res;
5242
5243     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5244     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5245
5246     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5247     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5248
5249     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5250     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5251
5252     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5253     return undef if !defined($product) || $product !~ s/^0x//;
5254
5255     $res = {
5256         name => $name,
5257         vendor => $vendor,
5258         product => $product,
5259         domain => $domain,
5260         bus => $bus,
5261         slot => $slot,
5262         func => $func,
5263         irq => $irq,
5264         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5265     };
5266
5267     return $res;
5268 }
5269
5270 sub pci_dev_reset {
5271     my ($dev) = @_;
5272
5273     my $name = $dev->{name};
5274
5275     my $fn = "$pcisysfs/devices/$name/reset";
5276
5277     return file_write($fn, "1");
5278 }
5279
5280 sub pci_dev_bind_to_vfio {
5281     my ($dev) = @_;
5282
5283     my $name = $dev->{name};
5284
5285     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5286
5287     if (!-d $vfio_basedir) {
5288         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5289     }
5290     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5291
5292     my $testdir = "$vfio_basedir/$name";
5293     return 1 if -d $testdir;
5294
5295     my $data = "$dev->{vendor} $dev->{product}";
5296     return undef if !file_write("$vfio_basedir/new_id", $data);
5297
5298     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5299     if (!file_write($fn, $name)) {
5300         return undef if -f $fn;
5301     }
5302
5303     $fn = "$vfio_basedir/bind";
5304     if (! -d $testdir) {
5305         return undef if !file_write($fn, $name);
5306     }
5307
5308     return -d $testdir;
5309 }
5310
5311 sub pci_dev_group_bind_to_vfio {
5312     my ($pciid) = @_;
5313
5314     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5315
5316     if (!-d $vfio_basedir) {
5317         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5318     }
5319     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5320
5321     # get IOMMU group devices
5322     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5323       my @devs = grep /^0000:/, readdir($D);
5324     closedir($D);
5325
5326     foreach my $pciid (@devs) {
5327         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5328
5329         # pci bridges, switches or root ports are not supported
5330         # they have a pci_bus subdirectory so skip them
5331         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5332
5333         my $info = pci_device_info($1);
5334         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5335     }
5336
5337     return 1;
5338 }
5339
5340 # vzdump restore implementaion
5341
5342 sub tar_archive_read_firstfile {
5343     my $archive = shift;
5344
5345     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5346
5347     # try to detect archive type first
5348     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5349         die "unable to open file '$archive'\n";
5350     my $firstfile = <$fh>;
5351     kill 15, $pid;
5352     close $fh;
5353
5354     die "ERROR: archive contaions no data\n" if !$firstfile;
5355     chomp $firstfile;
5356
5357     return $firstfile;
5358 }
5359
5360 sub tar_restore_cleanup {
5361     my ($storecfg, $statfile) = @_;
5362
5363     print STDERR "starting cleanup\n";
5364
5365     if (my $fd = IO::File->new($statfile, "r")) {
5366         while (defined(my $line = <$fd>)) {
5367             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5368                 my $volid = $2;
5369                 eval {
5370                     if ($volid =~ m|^/|) {
5371                         unlink $volid || die 'unlink failed\n';
5372                     } else {
5373                         PVE::Storage::vdisk_free($storecfg, $volid);
5374                     }
5375                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5376                 };
5377                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5378             } else {
5379                 print STDERR "unable to parse line in statfile - $line";
5380             }
5381         }
5382         $fd->close();
5383     }
5384 }
5385
5386 sub restore_archive {
5387     my ($archive, $vmid, $user, $opts) = @_;
5388
5389     my $format = $opts->{format};
5390     my $comp;
5391
5392     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5393         $format = 'tar' if !$format;
5394         $comp = 'gzip';
5395     } elsif ($archive =~ m/\.tar$/) {
5396         $format = 'tar' if !$format;
5397     } elsif ($archive =~ m/.tar.lzo$/) {
5398         $format = 'tar' if !$format;
5399         $comp = 'lzop';
5400     } elsif ($archive =~ m/\.vma$/) {
5401         $format = 'vma' if !$format;
5402     } elsif ($archive =~ m/\.vma\.gz$/) {
5403         $format = 'vma' if !$format;
5404         $comp = 'gzip';
5405     } elsif ($archive =~ m/\.vma\.lzo$/) {
5406         $format = 'vma' if !$format;
5407         $comp = 'lzop';
5408     } else {
5409         $format = 'vma' if !$format; # default
5410     }
5411
5412     # try to detect archive format
5413     if ($format eq 'tar') {
5414         return restore_tar_archive($archive, $vmid, $user, $opts);
5415     } else {
5416         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5417     }
5418 }
5419
5420 sub restore_update_config_line {
5421     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5422
5423     return if $line =~ m/^\#qmdump\#/;
5424     return if $line =~ m/^\#vzdump\#/;
5425     return if $line =~ m/^lock:/;
5426     return if $line =~ m/^unused\d+:/;
5427     return if $line =~ m/^parent:/;
5428     return if $line =~ m/^template:/; # restored VM is never a template
5429
5430     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5431     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5432         # try to convert old 1.X settings
5433         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5434         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5435             my ($model, $macaddr) = split(/\=/, $devconfig);
5436             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5437             my $net = {
5438                 model => $model,
5439                 bridge => "vmbr$ind",
5440                 macaddr => $macaddr,
5441             };
5442             my $netstr = print_net($net);
5443
5444             print $outfd "net$cookie->{netcount}: $netstr\n";
5445             $cookie->{netcount}++;
5446         }
5447     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5448         my ($id, $netstr) = ($1, $2);
5449         my $net = parse_net($netstr);
5450         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5451         $netstr = print_net($net);
5452         print $outfd "$id: $netstr\n";
5453     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5454         my $virtdev = $1;
5455         my $value = $3;
5456         my $di = parse_drive($virtdev, $value);
5457         if (defined($di->{backup}) && !$di->{backup}) {
5458             print $outfd "#$line";
5459         } elsif ($map->{$virtdev}) {
5460             delete $di->{format}; # format can change on restore
5461             $di->{file} = $map->{$virtdev};
5462             $value = print_drive($vmid, $di);
5463             print $outfd "$virtdev: $value\n";
5464         } else {
5465             print $outfd $line;
5466         }
5467     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5468         my ($uuid, $uuid_str);
5469         UUID::generate($uuid);
5470         UUID::unparse($uuid, $uuid_str);
5471         my $smbios1 = parse_smbios1($2);
5472         $smbios1->{uuid} = $uuid_str;
5473         print $outfd $1.print_smbios1($smbios1)."\n";
5474     } else {
5475         print $outfd $line;
5476     }
5477 }
5478
5479 sub scan_volids {
5480     my ($cfg, $vmid) = @_;
5481
5482     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5483
5484     my $volid_hash = {};
5485     foreach my $storeid (keys %$info) {
5486         foreach my $item (@{$info->{$storeid}}) {
5487             next if !($item->{volid} && $item->{size});
5488             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5489             $volid_hash->{$item->{volid}} = $item;
5490         }
5491     }
5492
5493     return $volid_hash;
5494 }
5495
5496 sub is_volume_in_use {
5497     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5498
5499     my $path = PVE::Storage::path($storecfg, $volid);
5500
5501     my $scan_config = sub {
5502         my ($cref, $snapname) = @_;
5503
5504         foreach my $key (keys %$cref) {
5505             my $value = $cref->{$key};
5506             if (is_valid_drivename($key)) {
5507                 next if $skip_drive && $key eq $skip_drive;
5508                 my $drive = parse_drive($key, $value);
5509                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5510                 return 1 if $volid eq $drive->{file};
5511                 if ($drive->{file} =~ m!^/!) {
5512                     return 1 if $drive->{file} eq $path;
5513                 } else {
5514                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5515                     next if !$storeid;
5516                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5517                     next if !$scfg;
5518                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5519                 }
5520             }
5521         }
5522
5523         return 0;
5524     };
5525
5526     return 1 if &$scan_config($conf);
5527
5528     undef $skip_drive;
5529
5530     foreach my $snapname (keys %{$conf->{snapshots}}) {
5531         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5532     }
5533
5534     return 0;
5535 }
5536
5537 sub update_disksize {
5538     my ($vmid, $conf, $volid_hash) = @_;
5539
5540     my $changes;
5541     my $prefix = "VM $vmid:";
5542
5543     # used and unused disks
5544     my $referenced = {};
5545
5546     # Note: it is allowed to define multiple storages with same path (alias), so
5547     # we need to check both 'volid' and real 'path' (two different volid can point
5548     # to the same path).
5549
5550     my $referencedpath = {};
5551
5552     # update size info
5553     foreach my $opt (keys %$conf) {
5554         if (is_valid_drivename($opt)) {
5555             my $drive = parse_drive($opt, $conf->{$opt});
5556             my $volid = $drive->{file};
5557             next if !$volid;
5558
5559             $referenced->{$volid} = 1;
5560             if ($volid_hash->{$volid} &&
5561                 (my $path = $volid_hash->{$volid}->{path})) {
5562                 $referencedpath->{$path} = 1;
5563             }
5564
5565             next if drive_is_cdrom($drive);
5566             next if !$volid_hash->{$volid};
5567
5568             $drive->{size} = $volid_hash->{$volid}->{size};
5569             my $new = print_drive($vmid, $drive);
5570             if ($new ne $conf->{$opt}) {
5571                 $changes = 1;
5572                 $conf->{$opt} = $new;
5573                 print "$prefix update disk '$opt' information.\n";
5574             }
5575         }
5576     }
5577
5578     # remove 'unusedX' entry if volume is used
5579     foreach my $opt (keys %$conf) {
5580         next if $opt !~ m/^unused\d+$/;
5581         my $volid = $conf->{$opt};
5582         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5583         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5584             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
5585             $changes = 1;
5586             delete $conf->{$opt};
5587         }
5588
5589         $referenced->{$volid} = 1;
5590         $referencedpath->{$path} = 1 if $path;
5591     }
5592
5593     foreach my $volid (sort keys %$volid_hash) {
5594         next if $volid =~ m/vm-$vmid-state-/;
5595         next if $referenced->{$volid};
5596         my $path = $volid_hash->{$volid}->{path};
5597         next if !$path; # just to be sure
5598         next if $referencedpath->{$path};
5599         $changes = 1;
5600         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5601         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
5602         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5603     }
5604
5605     return $changes;
5606 }
5607
5608 sub rescan {
5609     my ($vmid, $nolock, $dryrun) = @_;
5610
5611     my $cfg = PVE::Storage::config();
5612
5613     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5614     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5615     foreach my $stor (keys %{$cfg->{ids}}) {
5616         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5617     }
5618
5619     print "rescan volumes...\n";
5620     my $volid_hash = scan_volids($cfg, $vmid);
5621
5622     my $updatefn =  sub {
5623         my ($vmid) = @_;
5624
5625         my $conf = PVE::QemuConfig->load_config($vmid);
5626
5627         PVE::QemuConfig->check_lock($conf);
5628
5629         my $vm_volids = {};
5630         foreach my $volid (keys %$volid_hash) {
5631             my $info = $volid_hash->{$volid};
5632             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5633         }
5634
5635         my $changes = update_disksize($vmid, $conf, $vm_volids);
5636
5637         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5638     };
5639
5640     if (defined($vmid)) {
5641         if ($nolock) {
5642             &$updatefn($vmid);
5643         } else {
5644             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5645         }
5646     } else {
5647         my $vmlist = config_list();
5648         foreach my $vmid (keys %$vmlist) {
5649             if ($nolock) {
5650                 &$updatefn($vmid);
5651             } else {
5652                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5653             }
5654         }
5655     }
5656 }
5657
5658 sub restore_vma_archive {
5659     my ($archive, $vmid, $user, $opts, $comp) = @_;
5660
5661     my $readfrom = $archive;
5662
5663     my $cfg = PVE::Storage::config();
5664     my $commands = [];
5665     my $bwlimit = $opts->{bwlimit};
5666
5667     my $dbg_cmdstring = '';
5668     my $add_pipe = sub {
5669         my ($cmd) = @_;
5670         push @$commands, $cmd;
5671         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5672         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5673         $readfrom = '-';
5674     };
5675
5676     my $input = undef;
5677     if ($archive eq '-') {
5678         $input = '<&STDIN';
5679     } else {
5680         # If we use a backup from a PVE defined storage we also consider that
5681         # storage's rate limit:
5682         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5683         if (defined($volid)) {
5684             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5685             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5686             if ($readlimit) {
5687                 print STDERR "applying read rate limit: $readlimit\n";
5688                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5689                 $add_pipe->($cstream);
5690             }
5691         }
5692     }
5693
5694     if ($comp) {
5695         my $cmd;
5696         if ($comp eq 'gzip') {
5697             $cmd = ['zcat', $readfrom];
5698         } elsif ($comp eq 'lzop') {
5699             $cmd = ['lzop', '-d', '-c', $readfrom];
5700         } else {
5701             die "unknown compression method '$comp'\n";
5702         }
5703         $add_pipe->($cmd);
5704     }
5705
5706     my $tmpdir = "/var/tmp/vzdumptmp$$";
5707     rmtree $tmpdir;
5708
5709     # disable interrupts (always do cleanups)
5710     local $SIG{INT} =
5711         local $SIG{TERM} =
5712         local $SIG{QUIT} =
5713         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5714
5715     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5716     POSIX::mkfifo($mapfifo, 0600);
5717     my $fifofh;
5718
5719     my $openfifo = sub {
5720         open($fifofh, '>', $mapfifo) || die $!;
5721     };
5722
5723     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5724
5725     my $oldtimeout;
5726     my $timeout = 5;
5727
5728     my $devinfo = {};
5729
5730     my $rpcenv = PVE::RPCEnvironment::get();
5731
5732     my $conffile = PVE::QemuConfig->config_file($vmid);
5733     my $tmpfn = "$conffile.$$.tmp";
5734
5735     # Note: $oldconf is undef if VM does not exists
5736     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5737     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5738
5739     my %storage_limits;
5740
5741     my $print_devmap = sub {
5742         my $virtdev_hash = {};
5743
5744         my $cfgfn = "$tmpdir/qemu-server.conf";
5745
5746         # we can read the config - that is already extracted
5747         my $fh = IO::File->new($cfgfn, "r") ||
5748             "unable to read qemu-server.conf - $!\n";
5749
5750         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5751         if (-f $fwcfgfn) {
5752             my $pve_firewall_dir = '/etc/pve/firewall';
5753             mkdir $pve_firewall_dir; # make sure the dir exists
5754             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5755         }
5756
5757         while (defined(my $line = <$fh>)) {
5758             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5759                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5760                 die "archive does not contain data for drive '$virtdev'\n"
5761                     if !$devinfo->{$devname};
5762                 if (defined($opts->{storage})) {
5763                     $storeid = $opts->{storage} || 'local';
5764                 } elsif (!$storeid) {
5765                     $storeid = 'local';
5766                 }
5767                 $format = 'raw' if !$format;
5768                 $devinfo->{$devname}->{devname} = $devname;
5769                 $devinfo->{$devname}->{virtdev} = $virtdev;
5770                 $devinfo->{$devname}->{format} = $format;
5771                 $devinfo->{$devname}->{storeid} = $storeid;
5772
5773                 # check permission on storage
5774                 my $pool = $opts->{pool}; # todo: do we need that?
5775                 if ($user ne 'root@pam') {
5776                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5777                 }
5778
5779                 $storage_limits{$storeid} = $bwlimit;
5780
5781                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5782             }
5783         }
5784
5785         foreach my $key (keys %storage_limits) {
5786             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5787             next if !$limit;
5788             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5789             $storage_limits{$key} = $limit * 1024;
5790         }
5791
5792         foreach my $devname (keys %$devinfo) {
5793             die "found no device mapping information for device '$devname'\n"
5794                 if !$devinfo->{$devname}->{virtdev};
5795         }
5796
5797         # create empty/temp config
5798         if ($oldconf) {
5799             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5800             foreach_drive($oldconf, sub {
5801                 my ($ds, $drive) = @_;
5802
5803                 return if drive_is_cdrom($drive);
5804
5805                 my $volid = $drive->{file};
5806
5807                 return if !$volid || $volid =~ m|^/|;
5808
5809                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5810                 return if !$path || !$owner || ($owner != $vmid);
5811
5812                 # Note: only delete disk we want to restore
5813                 # other volumes will become unused
5814                 if ($virtdev_hash->{$ds}) {
5815                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5816                     if (my $err = $@) {
5817                         warn $err;
5818                     }
5819                 }
5820             });
5821
5822             # delete vmstate files
5823             # since after the restore we have no snapshots anymore
5824             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5825                 my $snap = $oldconf->{snapshots}->{$snapname};
5826                 if ($snap->{vmstate}) {
5827                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5828                     if (my $err = $@) {
5829                         warn $err;
5830                     }
5831                 }
5832             }
5833         }
5834
5835         my $map = {};
5836         foreach my $virtdev (sort keys %$virtdev_hash) {
5837             my $d = $virtdev_hash->{$virtdev};
5838             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5839             my $storeid = $d->{storeid};
5840             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5841
5842             my $map_opts = '';
5843             if (my $limit = $storage_limits{$storeid}) {
5844                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5845             }
5846
5847             # test if requested format is supported
5848             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5849             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5850             $d->{format} = $defFormat if !$supported;
5851
5852             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5853                                                   $d->{format}, undef, $alloc_size);
5854             print STDERR "new volume ID is '$volid'\n";
5855             $d->{volid} = $volid;
5856             my $path = PVE::Storage::path($cfg, $volid);
5857
5858             PVE::Storage::activate_volumes($cfg,[$volid]);
5859
5860             my $write_zeros = 1;
5861             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5862                 $write_zeros = 0;
5863             }
5864
5865             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5866
5867             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5868             $map->{$virtdev} = $volid;
5869         }
5870
5871         $fh->seek(0, 0) || die "seek failed - $!\n";
5872
5873         my $outfd = new IO::File ($tmpfn, "w") ||
5874             die "unable to write config for VM $vmid\n";
5875
5876         my $cookie = { netcount => 0 };
5877         while (defined(my $line = <$fh>)) {
5878             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5879         }
5880
5881         $fh->close();
5882         $outfd->close();
5883     };
5884
5885     eval {
5886         # enable interrupts
5887         local $SIG{INT} =
5888             local $SIG{TERM} =
5889             local $SIG{QUIT} =
5890             local $SIG{HUP} =
5891             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5892         local $SIG{ALRM} = sub { die "got timeout\n"; };
5893
5894         $oldtimeout = alarm($timeout);
5895
5896         my $parser = sub {
5897             my $line = shift;
5898
5899             print "$line\n";
5900
5901             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5902                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5903                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5904             } elsif ($line =~ m/^CTIME: /) {
5905                 # we correctly received the vma config, so we can disable
5906                 # the timeout now for disk allocation (set to 10 minutes, so
5907                 # that we always timeout if something goes wrong)
5908                 alarm(600);
5909                 &$print_devmap();
5910                 print $fifofh "done\n";
5911                 my $tmp = $oldtimeout || 0;
5912                 $oldtimeout = undef;
5913                 alarm($tmp);
5914                 close($fifofh);
5915             }
5916         };
5917
5918         print "restore vma archive: $dbg_cmdstring\n";
5919         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5920     };
5921     my $err = $@;
5922
5923     alarm($oldtimeout) if $oldtimeout;
5924
5925     my $vollist = [];
5926     foreach my $devname (keys %$devinfo) {
5927         my $volid = $devinfo->{$devname}->{volid};
5928         push @$vollist, $volid if $volid;
5929     }
5930
5931     PVE::Storage::deactivate_volumes($cfg, $vollist);
5932
5933     unlink $mapfifo;
5934
5935     if ($err) {
5936         rmtree $tmpdir;
5937         unlink $tmpfn;
5938
5939         foreach my $devname (keys %$devinfo) {
5940             my $volid = $devinfo->{$devname}->{volid};
5941             next if !$volid;
5942             eval {
5943                 if ($volid =~ m|^/|) {
5944                     unlink $volid || die 'unlink failed\n';
5945                 } else {
5946                     PVE::Storage::vdisk_free($cfg, $volid);
5947                 }
5948                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5949             };
5950             print STDERR "unable to cleanup '$volid' - $@" if $@;
5951         }
5952         die $err;
5953     }
5954
5955     rmtree $tmpdir;
5956
5957     rename($tmpfn, $conffile) ||
5958         die "unable to commit configuration file '$conffile'\n";
5959
5960     PVE::Cluster::cfs_update(); # make sure we read new file
5961
5962     eval { rescan($vmid, 1); };
5963     warn $@ if $@;
5964 }
5965
5966 sub restore_tar_archive {
5967     my ($archive, $vmid, $user, $opts) = @_;
5968
5969     if ($archive ne '-') {
5970         my $firstfile = tar_archive_read_firstfile($archive);
5971         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5972             if $firstfile ne 'qemu-server.conf';
5973     }
5974
5975     my $storecfg = PVE::Storage::config();
5976
5977     # destroy existing data - keep empty config
5978     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5979     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5980
5981     my $tocmd = "/usr/lib/qemu-server/qmextract";
5982
5983     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5984     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5985     $tocmd .= ' --prealloc' if $opts->{prealloc};
5986     $tocmd .= ' --info' if $opts->{info};
5987
5988     # tar option "xf" does not autodetect compression when read from STDIN,
5989     # so we pipe to zcat
5990     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5991         PVE::Tools::shellquote("--to-command=$tocmd");
5992
5993     my $tmpdir = "/var/tmp/vzdumptmp$$";
5994     mkpath $tmpdir;
5995
5996     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5997     local $ENV{VZDUMP_VMID} = $vmid;
5998     local $ENV{VZDUMP_USER} = $user;
5999
6000     my $conffile = PVE::QemuConfig->config_file($vmid);
6001     my $tmpfn = "$conffile.$$.tmp";
6002
6003     # disable interrupts (always do cleanups)
6004     local $SIG{INT} =
6005         local $SIG{TERM} =
6006         local $SIG{QUIT} =
6007         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6008
6009     eval {
6010         # enable interrupts
6011         local $SIG{INT} =
6012             local $SIG{TERM} =
6013             local $SIG{QUIT} =
6014             local $SIG{HUP} =
6015             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6016
6017         if ($archive eq '-') {
6018             print "extracting archive from STDIN\n";
6019             run_command($cmd, input => "<&STDIN");
6020         } else {
6021             print "extracting archive '$archive'\n";
6022             run_command($cmd);
6023         }
6024
6025         return if $opts->{info};
6026
6027         # read new mapping
6028         my $map = {};
6029         my $statfile = "$tmpdir/qmrestore.stat";
6030         if (my $fd = IO::File->new($statfile, "r")) {
6031             while (defined (my $line = <$fd>)) {
6032                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6033                     $map->{$1} = $2 if $1;
6034                 } else {
6035                     print STDERR "unable to parse line in statfile - $line\n";
6036                 }
6037             }
6038             $fd->close();
6039         }
6040
6041         my $confsrc = "$tmpdir/qemu-server.conf";
6042
6043         my $srcfd = new IO::File($confsrc, "r") ||
6044             die "unable to open file '$confsrc'\n";
6045
6046         my $outfd = new IO::File ($tmpfn, "w") ||
6047             die "unable to write config for VM $vmid\n";
6048
6049         my $cookie = { netcount => 0 };
6050         while (defined (my $line = <$srcfd>)) {
6051             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6052         }
6053
6054         $srcfd->close();
6055         $outfd->close();
6056     };
6057     my $err = $@;
6058
6059     if ($err) {
6060
6061         unlink $tmpfn;
6062
6063         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6064
6065         die $err;
6066     }
6067
6068     rmtree $tmpdir;
6069
6070     rename $tmpfn, $conffile ||
6071         die "unable to commit configuration file '$conffile'\n";
6072
6073     PVE::Cluster::cfs_update(); # make sure we read new file
6074
6075     eval { rescan($vmid, 1); };
6076     warn $@ if $@;
6077 };
6078
6079 sub foreach_storage_used_by_vm {
6080     my ($conf, $func) = @_;
6081
6082     my $sidhash = {};
6083
6084     foreach_drive($conf, sub {
6085         my ($ds, $drive) = @_;
6086         return if drive_is_cdrom($drive);
6087
6088         my $volid = $drive->{file};
6089
6090         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6091         $sidhash->{$sid} = $sid if $sid;
6092     });
6093
6094     foreach my $sid (sort keys %$sidhash) {
6095         &$func($sid);
6096     }
6097 }
6098
6099 sub do_snapshots_with_qemu {
6100     my ($storecfg, $volid) = @_;
6101
6102     my $storage_name = PVE::Storage::parse_volume_id($volid);
6103
6104     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6105         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6106         return 1;
6107     }
6108
6109     if ($volid =~ m/\.(qcow2|qed)$/){
6110         return 1;
6111     }
6112
6113     return undef;
6114 }
6115
6116 sub qga_check_running {
6117     my ($vmid, $nowarn) = @_;
6118
6119     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6120     if ($@) {
6121         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6122         return 0;
6123     }
6124     return 1;
6125 }
6126
6127 sub template_create {
6128     my ($vmid, $conf, $disk) = @_;
6129
6130     my $storecfg = PVE::Storage::config();
6131
6132     foreach_drive($conf, sub {
6133         my ($ds, $drive) = @_;
6134
6135         return if drive_is_cdrom($drive);
6136         return if $disk && $ds ne $disk;
6137
6138         my $volid = $drive->{file};
6139         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6140
6141         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6142         $drive->{file} = $voliddst;
6143         $conf->{$ds} = print_drive($vmid, $drive);
6144         PVE::QemuConfig->write_config($vmid, $conf);
6145     });
6146 }
6147
6148 sub qemu_img_convert {
6149     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6150
6151     my $storecfg = PVE::Storage::config();
6152     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6153     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6154
6155     if ($src_storeid && $dst_storeid) {
6156
6157         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6158
6159         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6160         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6161
6162         my $src_format = qemu_img_format($src_scfg, $src_volname);
6163         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6164
6165         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6166         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6167
6168         my $cmd = [];
6169         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6170         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6171         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6172         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6173         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6174         if ($is_zero_initialized) {
6175             push @$cmd, "zeroinit:$dst_path";
6176         } else {
6177             push @$cmd, $dst_path;
6178         }
6179
6180         my $parser = sub {
6181             my $line = shift;
6182             if($line =~ m/\((\S+)\/100\%\)/){
6183                 my $percent = $1;
6184                 my $transferred = int($size * $percent / 100);
6185                 my $remaining = $size - $transferred;
6186
6187                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6188             }
6189
6190         };
6191
6192         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6193         my $err = $@;
6194         die "copy failed: $err" if $err;
6195     }
6196 }
6197
6198 sub qemu_img_format {
6199     my ($scfg, $volname) = @_;
6200
6201     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6202         return $1;
6203     } else {
6204         return "raw";
6205     }
6206 }
6207
6208 sub qemu_drive_mirror {
6209     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6210
6211     $jobs = {} if !$jobs;
6212
6213     my $qemu_target;
6214     my $format;
6215     $jobs->{"drive-$drive"} = {};
6216
6217     if ($dst_volid =~ /^nbd:/) {
6218         $qemu_target = $dst_volid;
6219         $format = "nbd";
6220     } else {
6221         my $storecfg = PVE::Storage::config();
6222         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6223
6224         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6225
6226         $format = qemu_img_format($dst_scfg, $dst_volname);
6227
6228         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6229
6230         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6231     }
6232
6233     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6234     $opts->{format} = $format if $format;
6235
6236     print "drive mirror is starting for drive-$drive\n";
6237
6238     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6239
6240     if (my $err = $@) {
6241         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6242         die "mirroring error: $err";
6243     }
6244
6245     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6246 }
6247
6248 sub qemu_drive_mirror_monitor {
6249     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6250
6251     eval {
6252         my $err_complete = 0;
6253
6254         while (1) {
6255             die "storage migration timed out\n" if $err_complete > 300;
6256
6257             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6258
6259             my $running_mirror_jobs = {};
6260             foreach my $stat (@$stats) {
6261                 next if $stat->{type} ne 'mirror';
6262                 $running_mirror_jobs->{$stat->{device}} = $stat;
6263             }
6264
6265             my $readycounter = 0;
6266
6267             foreach my $job (keys %$jobs) {
6268
6269                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6270                     print "$job : finished\n";
6271                     delete $jobs->{$job};
6272                     next;
6273                 }
6274
6275                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6276
6277                 my $busy = $running_mirror_jobs->{$job}->{busy};
6278                 my $ready = $running_mirror_jobs->{$job}->{ready};
6279                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6280                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6281                     my $remaining = $total - $transferred;
6282                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6283
6284                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6285                 }
6286
6287                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6288             }
6289
6290             last if scalar(keys %$jobs) == 0;
6291
6292             if ($readycounter == scalar(keys %$jobs)) {
6293                 print "all mirroring jobs are ready \n";
6294                 last if $skipcomplete; #do the complete later
6295
6296                 if ($vmiddst && $vmiddst != $vmid) {
6297                     my $agent_running = $qga && qga_check_running($vmid);
6298                     if ($agent_running) {
6299                         print "freeze filesystem\n";
6300                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6301                     } else {
6302                         print "suspend vm\n";
6303                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6304                     }
6305
6306                     # if we clone a disk for a new target vm, we don't switch the disk
6307                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6308
6309                     if ($agent_running) {
6310                         print "unfreeze filesystem\n";
6311                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6312                     } else {
6313                         print "resume vm\n";
6314                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6315                     }
6316
6317                     last;
6318                 } else {
6319
6320                     foreach my $job (keys %$jobs) {
6321                         # try to switch the disk if source and destination are on the same guest
6322                         print "$job: Completing block job...\n";
6323
6324                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6325                         if ($@ =~ m/cannot be completed/) {
6326                             print "$job: Block job cannot be completed, try again.\n";
6327                             $err_complete++;
6328                         }else {
6329                             print "$job: Completed successfully.\n";
6330                             $jobs->{$job}->{complete} = 1;
6331                         }
6332                     }
6333                 }
6334             }
6335             sleep 1;
6336         }
6337     };
6338     my $err = $@;
6339
6340     if ($err) {
6341         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6342         die "mirroring error: $err";
6343     }
6344
6345 }
6346
6347 sub qemu_blockjobs_cancel {
6348     my ($vmid, $jobs) = @_;
6349
6350     foreach my $job (keys %$jobs) {
6351         print "$job: Cancelling block job\n";
6352         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6353         $jobs->{$job}->{cancel} = 1;
6354     }
6355
6356     while (1) {
6357         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6358
6359         my $running_jobs = {};
6360         foreach my $stat (@$stats) {
6361             $running_jobs->{$stat->{device}} = $stat;
6362         }
6363
6364         foreach my $job (keys %$jobs) {
6365
6366             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6367                 print "$job: Done.\n";
6368                 delete $jobs->{$job};
6369             }
6370         }
6371
6372         last if scalar(keys %$jobs) == 0;
6373
6374         sleep 1;
6375     }
6376 }
6377
6378 sub clone_disk {
6379     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6380         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6381
6382     my $newvolid;
6383
6384     if (!$full) {
6385         print "create linked clone of drive $drivename ($drive->{file})\n";
6386         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6387         push @$newvollist, $newvolid;
6388     } else {
6389
6390         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6391         $storeid = $storage if $storage;
6392
6393         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6394         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6395
6396         print "create full clone of drive $drivename ($drive->{file})\n";
6397         my $name = undef;
6398         if (drive_is_cloudinit($drive)) {
6399             $name = "vm-$newvmid-cloudinit";
6400             # cloudinit only supports raw and qcow2 atm:
6401             if ($dst_format eq 'qcow2') {
6402                 $name .= '.qcow2';
6403             } elsif ($dst_format ne 'raw') {
6404                 die "clone: unhandled format for cloudinit image\n";
6405             }
6406         }
6407         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6408         push @$newvollist, $newvolid;
6409
6410         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6411
6412         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6413         if (!$running || $snapname) {
6414             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6415         } else {
6416
6417             my $kvmver = get_running_qemu_version ($vmid);
6418             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6419                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6420                     if $drive->{iothread};
6421             }
6422
6423             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6424         }
6425     }
6426
6427     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6428
6429     my $disk = $drive;
6430     $disk->{format} = undef;
6431     $disk->{file} = $newvolid;
6432     $disk->{size} = $size;
6433
6434     return $disk;
6435 }
6436
6437 # this only works if VM is running
6438 sub get_current_qemu_machine {
6439     my ($vmid) = @_;
6440
6441     my $cmd = { execute => 'query-machines', arguments => {} };
6442     my $res = vm_qmp_command($vmid, $cmd);
6443
6444     my ($current, $default);
6445     foreach my $e (@$res) {
6446         $default = $e->{name} if $e->{'is-default'};
6447         $current = $e->{name} if $e->{'is-current'};
6448     }
6449
6450     # fallback to the default machine if current is not supported by qemu
6451     return $current || $default || 'pc';
6452 }
6453
6454 sub get_running_qemu_version {
6455     my ($vmid) = @_;
6456     my $cmd = { execute => 'query-version', arguments => {} };
6457     my $res = vm_qmp_command($vmid, $cmd);
6458     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6459 }
6460
6461 sub qemu_machine_feature_enabled {
6462     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6463
6464     my $current_major;
6465     my $current_minor;
6466
6467     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6468
6469         $current_major = $3;
6470         $current_minor = $4;
6471
6472     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6473
6474         $current_major = $1;
6475         $current_minor = $2;
6476     }
6477
6478     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6479
6480
6481 }
6482
6483 sub qemu_machine_pxe {
6484     my ($vmid, $conf, $machine) = @_;
6485
6486     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6487
6488     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6489         $machine .= '.pxe';
6490     }
6491
6492     return $machine;
6493 }
6494
6495 sub qemu_use_old_bios_files {
6496     my ($machine_type) = @_;
6497
6498     return if !$machine_type;
6499
6500     my $use_old_bios_files = undef;
6501
6502     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6503         $machine_type = $1;
6504         $use_old_bios_files = 1;
6505     } else {
6506         my $kvmver = kvm_user_version();
6507         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6508         # load new efi bios files on migration. So this hack is required to allow
6509         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6510         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6511         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6512     }
6513
6514     return ($use_old_bios_files, $machine_type);
6515 }
6516
6517 sub create_efidisk {
6518     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6519
6520     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6521
6522     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6523     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6524     PVE::Storage::activate_volumes($storecfg, [$volid]);
6525
6526     my $path = PVE::Storage::path($storecfg, $volid);
6527     eval {
6528         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6529     };
6530     die "Copying EFI vars image failed: $@" if $@;
6531
6532     return ($volid, $vars_size);
6533 }
6534
6535 sub lspci {
6536
6537     my $devices = {};
6538
6539     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6540             my (undef, $id, $function) = @_;
6541             my $res = { id => $id, function => $function};
6542             push @{$devices->{$id}}, $res;
6543     });
6544
6545     # Entries should be sorted by functions.
6546     foreach my $id (keys %$devices) {
6547         my $dev = $devices->{$id};
6548         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6549     }
6550
6551     return $devices;
6552 }
6553
6554 sub vm_iothreads_list {
6555     my ($vmid) = @_;
6556
6557     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6558
6559     my $iothreads = {};
6560     foreach my $iothread (@$res) {
6561         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6562     }
6563
6564     return $iothreads;
6565 }
6566
6567 sub scsihw_infos {
6568     my ($conf, $drive) = @_;
6569
6570     my $maxdev = 0;
6571
6572     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6573         $maxdev = 7;
6574     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6575         $maxdev = 1;
6576     } else {
6577         $maxdev = 256;
6578     }
6579
6580     my $controller = int($drive->{index} / $maxdev);
6581     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6582
6583     return ($maxdev, $controller, $controller_prefix);
6584 }
6585
6586 sub add_hyperv_enlightenments {
6587     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6588
6589     return if $winversion < 6;
6590     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6591
6592     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6593
6594     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6595         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6596         push @$cpuFlags , 'hv_vapic';
6597         push @$cpuFlags , 'hv_time';
6598     } else {
6599         push @$cpuFlags , 'hv_spinlocks=0xffff';
6600     }
6601
6602     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6603         push @$cpuFlags , 'hv_reset';
6604         push @$cpuFlags , 'hv_vpindex';
6605         push @$cpuFlags , 'hv_runtime';
6606     }
6607
6608     if ($winversion >= 7) {
6609         push @$cpuFlags , 'hv_relaxed';
6610     }
6611 }
6612
6613 sub windows_version {
6614     my ($ostype) = @_;
6615
6616     return 0 if !$ostype;
6617
6618     my $winversion = 0;
6619
6620     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6621         $winversion = 5;
6622     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6623         $winversion = 6;
6624     } elsif ($ostype =~ m/^win(\d+)$/) {
6625         $winversion = $1;
6626     }
6627
6628     return $winversion;
6629 }
6630
6631 sub resolve_dst_disk_format {
6632         my ($storecfg, $storeid, $src_volname, $format) = @_;
6633         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6634
6635         if (!$format) {
6636             # if no target format is specified, use the source disk format as hint
6637             if ($src_volname) {
6638                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6639                 $format = qemu_img_format($scfg, $src_volname);
6640             } else {
6641                 return $defFormat;
6642             }
6643         }
6644
6645         # test if requested format is supported - else use default
6646         my $supported = grep { $_ eq $format } @$validFormats;
6647         $format = $defFormat if !$supported;
6648         return $format;
6649 }
6650
6651 sub resolve_first_disk {
6652     my $conf = shift;
6653     my @disks = PVE::QemuServer::valid_drive_names();
6654     my $firstdisk;
6655     foreach my $ds (reverse @disks) {
6656         next if !$conf->{$ds};
6657         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6658         next if PVE::QemuServer::drive_is_cdrom($disk);
6659         $firstdisk = $ds;
6660     }
6661     return $firstdisk;
6662 }
6663
6664 sub generate_smbios1_uuid {
6665     my ($uuid, $uuid_str);
6666     UUID::generate($uuid);
6667     UUID::unparse($uuid, $uuid_str);
6668     return "uuid=$uuid_str";
6669 }
6670
6671 # bash completion helper
6672
6673 sub complete_backup_archives {
6674     my ($cmdname, $pname, $cvalue) = @_;
6675
6676     my $cfg = PVE::Storage::config();
6677
6678     my $storeid;
6679
6680     if ($cvalue =~ m/^([^:]+):/) {
6681         $storeid = $1;
6682     }
6683
6684     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6685
6686     my $res = [];
6687     foreach my $id (keys %$data) {
6688         foreach my $item (@{$data->{$id}}) {
6689             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6690             push @$res, $item->{volid} if defined($item->{volid});
6691         }
6692     }
6693
6694     return $res;
6695 }
6696
6697 my $complete_vmid_full = sub {
6698     my ($running) = @_;
6699
6700     my $idlist = vmstatus();
6701
6702     my $res = [];
6703
6704     foreach my $id (keys %$idlist) {
6705         my $d = $idlist->{$id};
6706         if (defined($running)) {
6707             next if $d->{template};
6708             next if $running && $d->{status} ne 'running';
6709             next if !$running && $d->{status} eq 'running';
6710         }
6711         push @$res, $id;
6712
6713     }
6714     return $res;
6715 };
6716
6717 sub complete_vmid {
6718     return &$complete_vmid_full();
6719 }
6720
6721 sub complete_vmid_stopped {
6722     return &$complete_vmid_full(0);
6723 }
6724
6725 sub complete_vmid_running {
6726     return &$complete_vmid_full(1);
6727 }
6728
6729 sub complete_storage {
6730
6731     my $cfg = PVE::Storage::config();
6732     my $ids = $cfg->{ids};
6733
6734     my $res = [];
6735     foreach my $sid (keys %$ids) {
6736         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6737         next if !$ids->{$sid}->{content}->{images};
6738         push @$res, $sid;
6739     }
6740
6741     return $res;
6742 }
6743
6744 sub nbd_stop {
6745     my ($vmid) = @_;
6746
6747     vm_mon_cmd($vmid, 'nbd-server-stop');
6748 }
6749
6750 1;