PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         default => 'kvm64',
 151         default_key => 1,
 152     },
 153     hidden => {
 154         description => "Do not identify as a KVM virtual machine.",
 155         type => 'boolean',
 156         optional => 1,
 157         default => 0
 158     },
 159 };
 160
 161 my $watchdog_fmt = {
 162     model => {
 163         default_key => 1,
 164         type => 'string',
 165         enum => [qw(i6300esb ib700)],
 166         description => "Watchdog type to emulate.",
 167         default => 'i6300esb',
 168         optional => 1,
 169     },
 170     action => {
 171         type => 'string',
 172         enum => [qw(reset shutdown poweroff pause debug none)],
 173         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 174         optional => 1,
 175     },
 176 };
 177 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 178
 179 my $confdesc = {
 180     onboot => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Specifies whether a VM will be started during system bootup.",
 184         default => 0,
 185     },
 186     autostart => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Automatic restart after crash (currently ignored).",
 190         default => 0,
 191     },
 192     hotplug => {
 193         optional => 1,
 194         type => 'string', format => 'pve-hotplug-features',
 195         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 196         default => 'network,disk,usb',
 197     },
 198     reboot => {
 199         optional => 1,
 200         type => 'boolean',
 201         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 202         default => 1,
 203     },
 204     lock => {
 205         optional => 1,
 206         type => 'string',
 207         description => "Lock/unlock the VM.",
 208         enum => [qw(migrate backup snapshot rollback)],
 209     },
 210     cpulimit => {
 211         optional => 1,
 212         type => 'number',
 213         description => "Limit of CPU usage.",
 214         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 215         minimum => 0,
 216         maximum => 128,
 217         default => 0,
 218     },
 219     cpuunits => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "CPU weight for a VM.",
 223         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 224         minimum => 0,
 225         maximum => 500000,
 226         default => 1024,
 227     },
 228     memory => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 232         minimum => 16,
 233         default => 512,
 234     },
 235     balloon => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 239         minimum => 0,
 240     },
 241     shares => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 245         minimum => 0,
 246         maximum => 50000,
 247         default => 1000,
 248     },
 249     keyboard => {
 250         optional => 1,
 251         type => 'string',
 252         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 253         enum => PVE::Tools::kvmkeymaplist(),
 254         default => 'en-us',
 255     },
 256     name => {
 257         optional => 1,
 258         type => 'string', format => 'dns-name',
 259         description => "Set a name for the VM. Only used on the configuration web interface.",
 260     },
 261     scsihw => {
 262         optional => 1,
 263         type => 'string',
 264         description => "SCSI controller model",
 265         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 266         default => 'lsi',
 267     },
 268     description => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 272     },
 273     ostype => {
 274         optional => 1,
 275         type => 'string',
 276         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 277         description => "Specify guest operating system.",
 278         verbose_description => <<EODESC,
 279 Specify guest operating system. This is used to enable special
 280 optimization/features for specific operating systems:
 281
 282 [horizontal]
 283 other;; unspecified OS
 284 wxp;; Microsoft Windows XP
 285 w2k;; Microsoft Windows 2000
 286 w2k3;; Microsoft Windows 2003
 287 w2k8;; Microsoft Windows 2008
 288 wvista;; Microsoft Windows Vista
 289 win7;; Microsoft Windows 7
 290 win8;; Microsoft Windows 8/2012
 291 l24;; Linux 2.4 Kernel
 292 l26;; Linux 2.6/3.X Kernel
 293 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 294 EODESC
 295     },
 296     boot => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 300         pattern => '[acdn]{1,4}',
 301         default => 'cdn',
 302     },
 303     bootdisk => {
 304         optional => 1,
 305         type => 'string', format => 'pve-qm-bootdisk',
 306         description => "Enable booting from specified disk.",
 307         pattern => '(ide|sata|scsi|virtio)\d+',
 308     },
 309     smp => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of CPUs. Please use option -sockets instead.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     sockets => {
 317         optional => 1,
 318         type => 'integer',
 319         description => "The number of CPU sockets.",
 320         minimum => 1,
 321         default => 1,
 322     },
 323     cores => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "The number of cores per socket.",
 327         minimum => 1,
 328         default => 1,
 329     },
 330     numa => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable NUMA.",
 334         default => 0,
 335     },
 336     hugepages => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Enable/disable hugepages memory.",
 340         enum => [qw(any 2 1024)],
 341     },
 342     vcpus => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Number of hotplugged vcpus.",
 346         minimum => 1,
 347         default => 0,
 348     },
 349     acpi => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable ACPI.",
 353         default => 1,
 354     },
 355     agent => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable Qemu GuestAgent.",
 359         default => 0,
 360     },
 361     kvm => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable KVM hardware virtualization.",
 365         default => 1,
 366     },
 367     tdf => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable time drift fix.",
 371         default => 0,
 372     },
 373     localtime => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 377     },
 378     freeze => {
 379         optional => 1,
 380         type => 'boolean',
 381         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 382     },
 383     vga => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Select the VGA type.",
 387         verbose_description => "Select the VGA type. If you want to use high resolution" .
 388             " modes (>= 1280x1024x16) then you should use the options " .
 389             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 390             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 391             "display sever. For win* OS you can select how many independent " .
 392             "displays you want, Linux guests can add displays them self. " .
 393             "You can also run without any graphic card, using a serial device" .
 394             " as terminal.",
 395         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 396     },
 397     watchdog => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-watchdog',
 400         description => "Create a virtual hardware watchdog device.",
 401         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 402             " (by a guest action), the watchdog must be periodically polled " .
 403             "by an agent inside the guest or else the watchdog will reset " .
 404             "the guest (or execute the respective action specified)",
 405     },
 406     startdate => {
 407         optional => 1,
 408         type => 'string',
 409         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 410         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 411         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 412         default => 'now',
 413     },
 414     startup =>  get_standard_option('pve-startup-order'),
 415     template => {
 416         optional => 1,
 417         type => 'boolean',
 418         description => "Enable/disable Template.",
 419         default => 0,
 420     },
 421     args => {
 422         optional => 1,
 423         type => 'string',
 424         description => "Arbitrary arguments passed to kvm.",
 425         verbose_description => <<EODESCR,
 426 Arbitrary arguments passed to kvm, for example:
 427
 428 args: -no-reboot -no-hpet
 429
 430 NOTE: this option is for experts only.
 431 EODESCR
 432     },
 433     tablet => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 1,
 437         description => "Enable/disable the USB tablet device.",
 438         verbose_description => "Enable/disable the USB tablet device. This device is " .
 439             "usually needed to allow absolute mouse positioning with VNC. " .
 440             "Else the mouse runs out of sync with normal VNC clients. " .
 441             "If you're running lots of console-only guests on one host, " .
 442             "you may consider disabling this to save some context switches. " .
 443             "This is turned off by default if you use spice (-vga=qxl).",
 444     },
 445     migrate_speed => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 449         minimum => 0,
 450         default => 0,
 451     },
 452     migrate_downtime => {
 453         optional => 1,
 454         type => 'number',
 455         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 456         minimum => 0,
 457         default => 0.1,
 458     },
 459     cdrom => {
 460         optional => 1,
 461         type => 'string', format => 'pve-qm-ide',
 462         typetext => '<volume>',
 463         description => "This is an alias for option -ide2",
 464     },
 465     cpu => {
 466         optional => 1,
 467         description => "Emulated CPU type.",
 468         type => 'string',
 469         format => $cpu_fmt,
 470     },
 471     parent => get_standard_option('pve-snapshot-name', {
 472         optional => 1,
 473         description => "Parent snapshot name. This is used internally, and should not be modified.",
 474     }),
 475     snaptime => {
 476         optional => 1,
 477         description => "Timestamp for snapshots.",
 478         type => 'integer',
 479         minimum => 0,
 480     },
 481     vmstate => {
 482         optional => 1,
 483         type => 'string', format => 'pve-volume-id',
 484         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 485     },
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     werror => {
 756         type => 'string',
 757         enum => [qw(enospc ignore report stop)],
 758         description => 'Write error action.',
 759         optional => 1,
 760     },
 761     aio => {
 762         type => 'string',
 763         enum => [qw(native threads)],
 764         description => 'AIO type to use.',
 765         optional => 1,
 766     },
 767     discard => {
 768         type => 'string',
 769         enum => [qw(ignore on)],
 770         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 771         optional => 1,
 772     },
 773     detect_zeroes => {
 774         type => 'boolean',
 775         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 776         optional => 1,
 777     },
 778     serial => {
 779         type => 'string',
 780         format => 'urlencoded',
 781         format_description => 'serial',
 782         maxLength => 20*3, # *3 since it's %xx url enoded
 783         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 784         optional => 1,
 785     }
 786 );
 787
 788 my %rerror_fmt = (
 789     rerror => {
 790         type => 'string',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         description => "Whether to use iothreads for this drive",
 800         optional => 1,
 801 });
 802
 803 my %model_fmt = (
 804     model => {
 805         type => 'string',
 806         format => 'urlencoded',
 807         format_description => 'model',
 808         maxLength => 40*3, # *3 since it's %xx url enoded
 809         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 810         optional => 1,
 811     },
 812 );
 813
 814 my %queues_fmt = (
 815     queues => {
 816         type => 'integer',
 817         description => "Number of queues.",
 818         minimum => 2,
 819         optional => 1
 820     }
 821 );
 822
 823 my %scsiblock_fmt = (
 824     scsiblock => {
 825         type => 'boolean',
 826         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 827         optional => 1,
 828         default => 0,
 829     },
 830 );
 831
 832 my $add_throttle_desc = sub {
 833     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 834     my $d = {
 835         type => $type,
 836         format_description => $unit,
 837         description => "Maximum $what in $longunit.",
 838         optional => 1,
 839     };
 840     $d->{minimum} = $minimum if defined($minimum);
 841     $drivedesc_base{$key} = $d;
 842 };
 843 # throughput: (leaky bucket)
 844 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 845 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 846 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 847 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 850 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 851 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 852 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 853
 854 # pools: (pool of IO before throttling starts taking effect)
 855 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 856 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 857 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 858 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 859 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 860 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 861
 862 # burst lengths
 863 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 864 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 865 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 869
 870 my $ide_fmt = {
 871     %drivedesc_base,
 872     %rerror_fmt,
 873     %model_fmt,
 874 };
 875 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 876
 877 my $idedesc = {
 878     optional => 1,
 879     type => 'string', format => $ide_fmt,
 880     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 883
 884 my $scsi_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %queues_fmt,
 888     %scsiblock_fmt,
 889 };
 890 my $scsidesc = {
 891     optional => 1,
 892     type => 'string', format => $scsi_fmt,
 893     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 894 };
 895 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 896
 897 my $sata_fmt = {
 898     %drivedesc_base,
 899     %rerror_fmt,
 900 };
 901 my $satadesc = {
 902     optional => 1,
 903     type => 'string', format => $sata_fmt,
 904     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 905 };
 906 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 907
 908 my $virtio_fmt = {
 909     %drivedesc_base,
 910     %iothread_fmt,
 911     %rerror_fmt,
 912 };
 913 my $virtiodesc = {
 914     optional => 1,
 915     type => 'string', format => $virtio_fmt,
 916     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 917 };
 918 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 919
 920 my $alldrive_fmt = {
 921     %drivedesc_base,
 922     %rerror_fmt,
 923     %iothread_fmt,
 924     %model_fmt,
 925     %queues_fmt,
 926     %scsiblock_fmt,
 927 };
 928
 929 my $efidisk_fmt = {
 930     volume => { alias => 'file' },
 931     file => {
 932         type => 'string',
 933         format => 'pve-volume-id-or-qm-path',
 934         default_key => 1,
 935         format_description => 'volume',
 936         description => "The drive's backing volume.",
 937     },
 938     format => get_standard_option('pve-qm-image-format'),
 939     size => {
 940         type => 'string',
 941         format => 'disk-size',
 942         format_description => 'DiskSize',
 943         description => "Disk size. This is purely informational and has no effect.",
 944         optional => 1,
 945     },
 946 };
 947
 948 my $efidisk_desc = {
 949     optional => 1,
 950     type => 'string', format => $efidisk_fmt,
 951     description => "Configure a Disk for storing EFI vars",
 952 };
 953
 954 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 955
 956 my $usb_fmt = {
 957     host => {
 958         default_key => 1,
 959         type => 'string', format => 'pve-qm-usb-device',
 960         format_description => 'HOSTUSBDEVICE|spice',
 961         description => <<EODESCR,
 962 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 963
 964  'bus-port(.port)*' (decimal numbers) or
 965  'vendor_id:product_id' (hexadeciaml numbers) or
 966  'spice'
 967
 968 You can use the 'lsusb -t' command to list existing usb devices.
 969
 970 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 971
 972 The value 'spice' can be used to add a usb redirection devices for spice.
 973 EODESCR
 974     },
 975     usb3 => {
 976         optional => 1,
 977         type => 'boolean',
 978         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 979         default => 0,
 980     },
 981 };
 982
 983 my $usbdesc = {
 984     optional => 1,
 985     type => 'string', format => $usb_fmt,
 986     description => "Configure an USB device (n is 0 to 4).",
 987 };
 988 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 989
 990 # NOTE: the match-groups of this regex are used in parse_hostpci
 991 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 992 my $hostpci_fmt = {
 993     host => {
 994         default_key => 1,
 995         type => 'string',
 996         pattern => qr/$PCIRE(;$PCIRE)*/,
 997         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 998         description => <<EODESCR,
 999 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1000 of PCI virtual functions of the host. HOSTPCIID syntax is:
1001
1002 'bus:dev.func' (hexadecimal numbers)
1003
1004 You can us the 'lspci' command to list existing PCI devices.
1005 EODESCR
1006     },
1007     rombar => {
1008         type => 'boolean',
1009         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1010         optional => 1,
1011         default => 1,
1012     },
1013     romfile => {
1014         type => 'string',
1015         pattern => '[^,;]+',
1016         format_description => 'string',
1017         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1018         optional => 1,
1019     },
1020     pcie => {
1021         type => 'boolean',
1022         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1023         optional => 1,
1024         default => 0,
1025     },
1026     'x-vga' => {
1027         type => 'boolean',
1028         description =>  "Enable vfio-vga device support.",
1029         optional => 1,
1030         default => 0,
1031     },
1032 };
1033 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1034
1035 my $hostpcidesc = {
1036         optional => 1,
1037         type => 'string', format => 'pve-qm-hostpci',
1038         description => "Map host PCI devices into guest.",
1039         verbose_description =>  <<EODESCR,
1040 Map host PCI devices into guest.
1041
1042 NOTE: This option allows direct access to host hardware. So it is no longer
1043 possible to migrate such machines - use with special care.
1044
1045 CAUTION: Experimental! User reported problems with this option.
1046 EODESCR
1047 };
1048 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1049
1050 my $serialdesc = {
1051         optional => 1,
1052         type => 'string',
1053         pattern => '(/dev/.+|socket)',
1054         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1055         verbose_description =>  <<EODESCR,
1056 Create a serial device inside the VM (n is 0 to 3), and pass through a
1057 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1058 host side (use 'qm terminal' to open a terminal connection).
1059
1060 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1061
1062 CAUTION: Experimental! User reported problems with this option.
1063 EODESCR
1064 };
1065
1066 my $paralleldesc= {
1067         optional => 1,
1068         type => 'string',
1069         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1070         description =>  "Map host parallel devices (n is 0 to 2).",
1071         verbose_description =>  <<EODESCR,
1072 Map host parallel devices (n is 0 to 2).
1073
1074 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1081     $confdesc->{"parallel$i"} = $paralleldesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1085     $confdesc->{"serial$i"} = $serialdesc;
1086 }
1087
1088 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1089     $confdesc->{"hostpci$i"} = $hostpcidesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1093     $drivename_hash->{"ide$i"} = 1;
1094     $confdesc->{"ide$i"} = $idedesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1098     $drivename_hash->{"sata$i"} = 1;
1099     $confdesc->{"sata$i"} = $satadesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1103     $drivename_hash->{"scsi$i"} = 1;
1104     $confdesc->{"scsi$i"} = $scsidesc ;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1108     $drivename_hash->{"virtio$i"} = 1;
1109     $confdesc->{"virtio$i"} = $virtiodesc;
1110 }
1111
1112 $drivename_hash->{efidisk0} = 1;
1113 $confdesc->{efidisk0} = $efidisk_desc;
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $unuseddesc = {
1120     optional => 1,
1121     type => 'string', format => 'pve-volume-id',
1122     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1123 };
1124
1125 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1126     $confdesc->{"unused$i"} = $unuseddesc;
1127 }
1128
1129 my $kvm_api_version = 0;
1130
1131 sub kvm_version {
1132
1133     return $kvm_api_version if $kvm_api_version;
1134
1135     my $fh = IO::File->new("</dev/kvm") ||
1136         return 0;
1137
1138     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1139         $kvm_api_version = $v;
1140     }
1141
1142     $fh->close();
1143
1144     return  $kvm_api_version;
1145 }
1146
1147 my $kvm_user_version;
1148
1149 sub kvm_user_version {
1150
1151     return $kvm_user_version if $kvm_user_version;
1152
1153     $kvm_user_version = 'unknown';
1154
1155     my $code = sub {
1156         my $line = shift;
1157         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1158             $kvm_user_version = $2;
1159         }
1160     };
1161
1162     eval { run_command("kvm -version", outfunc => $code); };
1163     warn $@ if $@;
1164
1165     return $kvm_user_version;
1166
1167 }
1168
1169 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1170
1171 sub valid_drive_names {
1172     # order is important - used to autoselect boot disk
1173     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1174             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1175             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1176             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1177             'efidisk0');
1178 }
1179
1180 sub is_valid_drivename {
1181     my $dev = shift;
1182
1183     return defined($drivename_hash->{$dev});
1184 }
1185
1186 sub option_exists {
1187     my $key = shift;
1188     return defined($confdesc->{$key});
1189 }
1190
1191 sub nic_models {
1192     return $nic_model_list;
1193 }
1194
1195 sub os_list_description {
1196
1197     return {
1198         other => 'Other',
1199         wxp => 'Windows XP',
1200         w2k => 'Windows 2000',
1201         w2k3 =>, 'Windows 2003',
1202         w2k8 => 'Windows 2008',
1203         wvista => 'Windows Vista',
1204         win7 => 'Windows 7',
1205         win8 => 'Windows 8/2012',
1206         win10 => 'Windows 10/2016',
1207         l24 => 'Linux 2.4',
1208         l26 => 'Linux 2.6',
1209     };
1210 }
1211
1212 my $cdrom_path;
1213
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241     if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return undef if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1286         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1287         verify_media_type($opt, $vtype, $drive->{media});
1288         $drive->{file} = $volid;
1289     }
1290
1291     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1292 }
1293
1294 sub parse_hotplug_features {
1295     my ($data) = @_;
1296
1297     my $res = {};
1298
1299     return $res if $data eq '0';
1300
1301     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1302
1303     foreach my $feature (PVE::Tools::split_list($data)) {
1304         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1305             $res->{$1} = 1;
1306         } else {
1307             die "invalid hotplug feature '$feature'\n";
1308         }
1309     }
1310     return $res;
1311 }
1312
1313 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1314 sub pve_verify_hotplug_features {
1315     my ($value, $noerr) = @_;
1316
1317     return $value if parse_hotplug_features($value);
1318
1319     return undef if $noerr;
1320
1321     die "unable to parse hotplug option\n";
1322 }
1323
1324 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1325 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1326 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1327 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1328 #        [,iothread=on][,serial=serial][,model=model]
1329
1330 sub parse_drive {
1331     my ($key, $data) = @_;
1332
1333     my ($interface, $index);
1334
1335     if ($key =~ m/^([^\d]+)(\d+)$/) {
1336         $interface = $1;
1337         $index = $2;
1338     } else {
1339         return undef;
1340     }
1341
1342     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1343                                      : $confdesc->{$key}->{format};
1344     if (!$desc) {
1345         warn "invalid drive key: $key\n";
1346         return undef;
1347     }
1348     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1349     return undef if !$res;
1350     $res->{interface} = $interface;
1351     $res->{index} = $index;
1352
1353     my $error = 0;
1354     foreach my $opt (qw(bps bps_rd bps_wr)) {
1355         if (my $bps = defined(delete $res->{$opt})) {
1356             if (defined($res->{"m$opt"})) {
1357                 warn "both $opt and m$opt specified\n";
1358                 ++$error;
1359                 next;
1360             }
1361             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1362         }
1363     }
1364
1365     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1366     for my $requirement (
1367         [bps_max_length => 'mbps_max'],
1368         [bps_rd_max_length => 'mbps_rd_max'],
1369         [bps_wr_max_length => 'mbps_wr_max'],
1370         [iops_max_length => 'iops_max'],
1371         [iops_rd_max_length => 'iops_rd_max'],
1372         [iops_wr_max_length => 'iops_wr_max']) {
1373         my ($option, $requires) = @$requirement;
1374         if ($res->{$option} && !$res->{$requires}) {
1375             warn "$option requires $requires\n";
1376             ++$error;
1377         }
1378     }
1379
1380     return undef if $error;
1381
1382     return undef if $res->{mbps_rd} && $res->{mbps};
1383     return undef if $res->{mbps_wr} && $res->{mbps};
1384     return undef if $res->{iops_rd} && $res->{iops};
1385     return undef if $res->{iops_wr} && $res->{iops};
1386
1387     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1388         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1389         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1390         return undef if $res->{interface} eq 'virtio';
1391     }
1392
1393     if (my $size = $res->{size}) {
1394         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1395     }
1396
1397     return $res;
1398 }
1399
1400 sub print_drive {
1401     my ($vmid, $drive) = @_;
1402     my $data = { %$drive };
1403     delete $data->{$_} for qw(index interface);
1404     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1405 }
1406
1407 sub scsi_inquiry {
1408     my($fh, $noerr) = @_;
1409
1410     my $SG_IO = 0x2285;
1411     my $SG_GET_VERSION_NUM = 0x2282;
1412
1413     my $versionbuf = "\x00" x 8;
1414     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1415     if (!$ret) {
1416         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1417         return undef;
1418     }
1419     my $version = unpack("I", $versionbuf);
1420     if ($version < 30000) {
1421         die "scsi generic interface too old\n"  if !$noerr;
1422         return undef;
1423     }
1424
1425     my $buf = "\x00" x 36;
1426     my $sensebuf = "\x00" x 8;
1427     my $cmd = pack("C x3 C x1", 0x12, 36);
1428
1429     # see /usr/include/scsi/sg.h
1430     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1431
1432     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1433                       length($sensebuf), 0, length($buf), $buf,
1434                       $cmd, $sensebuf, 6000);
1435
1436     $ret = ioctl($fh, $SG_IO, $packet);
1437     if (!$ret) {
1438         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1439         return undef;
1440     }
1441
1442     my @res = unpack($sg_io_hdr_t, $packet);
1443     if ($res[17] || $res[18]) {
1444         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1445         return undef;
1446     }
1447
1448     my $res = {};
1449     (my $byte0, my $byte1, $res->{vendor},
1450      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1451
1452     $res->{removable} = $byte1 & 128 ? 1 : 0;
1453     $res->{type} = $byte0 & 31;
1454
1455     return $res;
1456 }
1457
1458 sub path_is_scsi {
1459     my ($path) = @_;
1460
1461     my $fh = IO::File->new("+<$path") || return undef;
1462     my $res = scsi_inquiry($fh, 1);
1463     close($fh);
1464
1465     return $res;
1466 }
1467
1468 sub machine_type_is_q35 {
1469     my ($conf) = @_;
1470
1471     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1472 }
1473
1474 sub print_tabletdevice_full {
1475     my ($conf) = @_;
1476
1477     my $q35 = machine_type_is_q35($conf);
1478
1479     # we use uhci for old VMs because tablet driver was buggy in older qemu
1480     my $usbbus = $q35 ? "ehci" : "uhci";
1481
1482     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1483 }
1484
1485 sub print_drivedevice_full {
1486     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1487
1488     my $device = '';
1489     my $maxdev = 0;
1490
1491     if ($drive->{interface} eq 'virtio') {
1492         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1493         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1494         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1495     } elsif ($drive->{interface} eq 'scsi') {
1496
1497         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1498         my $unit = $drive->{index} % $maxdev;
1499         my $devicetype = 'hd';
1500         my $path = '';
1501         if (drive_is_cdrom($drive)) {
1502             $devicetype = 'cd';
1503         } else {
1504             if ($drive->{file} =~ m|^/|) {
1505                 $path = $drive->{file};
1506                 if (my $info = path_is_scsi($path)) {
1507                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1508                         $devicetype = 'block';
1509                     } elsif ($info->{type} == 1) { # tape
1510                         $devicetype = 'generic';
1511                     }
1512                 }
1513             } else {
1514                  $path = PVE::Storage::path($storecfg, $drive->{file});
1515             }
1516
1517             if($path =~ m/^iscsi\:\/\//){
1518                 $devicetype = 'generic';
1519             }
1520         }
1521
1522         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1523             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1524         } else {
1525             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1526         }
1527
1528     } elsif ($drive->{interface} eq 'ide'){
1529         $maxdev = 2;
1530         my $controller = int($drive->{index} / $maxdev);
1531         my $unit = $drive->{index} % $maxdev;
1532         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1533
1534         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1536             $model = URI::Escape::uri_unescape($model);
1537             $device .= ",model=$model";
1538         }
1539     } elsif ($drive->{interface} eq 'sata'){
1540         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1541         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1542         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1543     } elsif ($drive->{interface} eq 'usb') {
1544         die "implement me";
1545         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1546     } else {
1547         die "unsupported interface type";
1548     }
1549
1550     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1551
1552     return $device;
1553 }
1554
1555 sub get_initiator_name {
1556     my $initiator;
1557
1558     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1559     while (defined(my $line = <$fh>)) {
1560         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1561         $initiator = $1;
1562         last;
1563     }
1564     $fh->close();
1565
1566     return $initiator;
1567 }
1568
1569 sub print_drive_full {
1570     my ($storecfg, $vmid, $drive) = @_;
1571
1572     my $path;
1573     my $volid = $drive->{file};
1574     my $format;
1575
1576     if (drive_is_cdrom($drive)) {
1577         $path = get_iso_path($storecfg, $vmid, $volid);
1578     } else {
1579         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1580         if ($storeid) {
1581             $path = PVE::Storage::path($storecfg, $volid);
1582             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1583             $format = qemu_img_format($scfg, $volname);
1584         } else {
1585             $path = $volid;
1586             $format = "raw";
1587         }
1588    }
1589
1590     my $opts = '';
1591     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1592     foreach my $o (@qemu_drive_options) {
1593         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1594     }
1595     if (my $serial = $drive->{serial}) {
1596         $serial = URI::Escape::uri_unescape($serial);
1597         $opts .= ",serial=$serial";
1598     }
1599
1600     $opts .= ",format=$format" if $format && !$drive->{format};
1601
1602     foreach my $o (qw(bps bps_rd bps_wr)) {
1603         my $v = $drive->{"m$o"};
1604         $opts .= ",$o=" . int($v*1024*1024) if $v;
1605     }
1606
1607     my $cache_direct = 0;
1608
1609     if (my $cache = $drive->{cache}) {
1610         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1611     } elsif (!drive_is_cdrom($drive)) {
1612         $opts .= ",cache=none";
1613         $cache_direct = 1;
1614     }
1615
1616     # aio native works only with O_DIRECT
1617     if (!$drive->{aio}) {
1618         if($cache_direct) {
1619             $opts .= ",aio=native";
1620         } else {
1621             $opts .= ",aio=threads";
1622         }
1623     }
1624
1625     if (!drive_is_cdrom($drive)) {
1626         my $detectzeroes;
1627         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1628             $detectzeroes = 'off';
1629         } elsif ($drive->{discard}) {
1630             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1631         } else {
1632             # This used to be our default with discard not being specified:
1633             $detectzeroes = 'on';
1634         }
1635         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1636     }
1637
1638     my $pathinfo = $path ? "file=$path," : '';
1639
1640     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1641 }
1642
1643 sub print_netdevice_full {
1644     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1645
1646     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1647
1648     my $device = $net->{model};
1649     if ($net->{model} eq 'virtio') {
1650          $device = 'virtio-net-pci';
1651      };
1652
1653     my $pciaddr = print_pci_addr("$netid", $bridges);
1654     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1655     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1656         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1657         my $vectors = $net->{queues} * 2 + 2;
1658         $tmpstr .= ",vectors=$vectors,mq=on";
1659     }
1660     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1661
1662     if ($use_old_bios_files) {
1663         my $romfile;
1664         if ($device eq 'virtio-net-pci') {
1665             $romfile = 'pxe-virtio.rom';
1666         } elsif ($device eq 'e1000') {
1667             $romfile = 'pxe-e1000.rom';
1668         } elsif ($device eq 'ne2k') {
1669             $romfile = 'pxe-ne2k_pci.rom';
1670         } elsif ($device eq 'pcnet') {
1671             $romfile = 'pxe-pcnet.rom';
1672         } elsif ($device eq 'rtl8139') {
1673             $romfile = 'pxe-rtl8139.rom';
1674         }
1675         $tmpstr .= ",romfile=$romfile" if $romfile;
1676     }
1677
1678     return $tmpstr;
1679 }
1680
1681 sub print_netdev_full {
1682     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1683
1684     my $i = '';
1685     if ($netid =~ m/^net(\d+)$/) {
1686         $i = int($1);
1687     }
1688
1689     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1690
1691     my $ifname = "tap${vmid}i$i";
1692
1693     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1694     die "interface name '$ifname' is too long (max 15 character)\n"
1695         if length($ifname) >= 16;
1696
1697     my $vhostparam = '';
1698     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1699
1700     my $vmname = $conf->{name} || "vm$vmid";
1701
1702     my $netdev = "";
1703     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1704
1705     if ($net->{bridge}) {
1706         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1707     } else {
1708         $netdev = "type=user,id=$netid,hostname=$vmname";
1709     }
1710
1711     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1712
1713     return $netdev;
1714 }
1715
1716
1717 sub print_cpu_device {
1718     my ($conf, $id) = @_;
1719
1720     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1721     my $cpu = $nokvm ? "qemu64" : "kvm64";
1722     if (my $cputype = $conf->{cpu}) {
1723         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1724             or die "Cannot parse cpu description: $cputype\n";
1725         $cpu = $cpuconf->{cputype};
1726     }
1727
1728     my $sockets = 1;
1729     $sockets = $conf->{sockets} if  $conf->{sockets};
1730     my $cores = $conf->{cores} || 1;
1731
1732     my $current_core = ($id - 1) % $cores;
1733     my $current_socket = int(($id - $current_core)/$cores);
1734
1735     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1736 }
1737
1738 sub drive_is_cdrom {
1739     my ($drive) = @_;
1740
1741     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1742
1743 }
1744
1745 sub parse_number_sets {
1746     my ($set) = @_;
1747     my $res = [];
1748     foreach my $part (split(/;/, $set)) {
1749         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1750             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1751             push @$res, [ $1, $2 ];
1752         } else {
1753             die "invalid range: $part\n";
1754         }
1755     }
1756     return $res;
1757 }
1758
1759 sub parse_numa {
1760     my ($data) = @_;
1761
1762     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1763     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1764     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1765     return $res;
1766 }
1767
1768 sub parse_hostpci {
1769     my ($value) = @_;
1770
1771     return undef if !$value;
1772
1773     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1774
1775     my @idlist = split(/;/, $res->{host});
1776     delete $res->{host};
1777     foreach my $id (@idlist) {
1778         if ($id =~ /^$PCIRE$/) {
1779             if (defined($2)) {
1780                 push @{$res->{pciid}}, { id => $1, function => $2 };
1781             } else {
1782                 my $pcidevices = lspci($1);
1783                 $res->{pciid} = $pcidevices->{$1};
1784             }
1785         } else {
1786             # should have been caught by parse_property_string already
1787             die "failed to parse PCI id: $id\n";
1788         }
1789     }
1790     return $res;
1791 }
1792
1793 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1794 sub parse_net {
1795     my ($data) = @_;
1796
1797     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1798     if ($@) {
1799         warn $@;
1800         return undef;
1801     }
1802     if (!defined($res->{macaddr})) {
1803         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1804         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1805     }
1806     return $res;
1807 }
1808
1809 sub print_net {
1810     my $net = shift;
1811
1812     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1813 }
1814
1815 sub add_random_macs {
1816     my ($settings) = @_;
1817
1818     foreach my $opt (keys %$settings) {
1819         next if $opt !~ m/^net(\d+)$/;
1820         my $net = parse_net($settings->{$opt});
1821         next if !$net;
1822         $settings->{$opt} = print_net($net);
1823     }
1824 }
1825
1826 sub vm_is_volid_owner {
1827     my ($storecfg, $vmid, $volid) = @_;
1828
1829     if ($volid !~  m|^/|) {
1830         my ($path, $owner);
1831         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1832         if ($owner && ($owner == $vmid)) {
1833             return 1;
1834         }
1835     }
1836
1837     return undef;
1838 }
1839
1840 sub split_flagged_list {
1841     my $text = shift || '';
1842     $text =~ s/[,;]/ /g;
1843     $text =~ s/^\s+//;
1844     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1845 }
1846
1847 sub join_flagged_list {
1848     my ($how, $lst) = @_;
1849     join $how, map { $lst->{$_} . $_ } keys %$lst;
1850 }
1851
1852 sub vmconfig_delete_pending_option {
1853     my ($conf, $key, $force) = @_;
1854
1855     delete $conf->{pending}->{$key};
1856     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1857     $pending_delete_hash->{$key} = $force ? '!' : '';
1858     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1859 }
1860
1861 sub vmconfig_undelete_pending_option {
1862     my ($conf, $key) = @_;
1863
1864     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1865     delete $pending_delete_hash->{$key};
1866
1867     if (%$pending_delete_hash) {
1868         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1869     } else {
1870         delete $conf->{pending}->{delete};
1871     }
1872 }
1873
1874 sub vmconfig_register_unused_drive {
1875     my ($storecfg, $vmid, $conf, $drive) = @_;
1876
1877     if (!drive_is_cdrom($drive)) {
1878         my $volid = $drive->{file};
1879         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1880             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1881         }
1882     }
1883 }
1884
1885 sub vmconfig_cleanup_pending {
1886     my ($conf) = @_;
1887
1888     # remove pending changes when nothing changed
1889     my $changes;
1890     foreach my $opt (keys %{$conf->{pending}}) {
1891         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1892             $changes = 1;
1893             delete $conf->{pending}->{$opt};
1894         }
1895     }
1896
1897     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1898     my $pending_delete_hash = {};
1899     while (my ($opt, $force) = each %$current_delete_hash) {
1900         if (defined($conf->{$opt})) {
1901             $pending_delete_hash->{$opt} = $force;
1902         } else {
1903             $changes = 1;
1904         }
1905     }
1906
1907     if (%$pending_delete_hash) {
1908         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1909     } else {
1910         delete $conf->{pending}->{delete};
1911     }
1912
1913     return $changes;
1914 }
1915
1916 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1917 my $smbios1_fmt = {
1918     uuid => {
1919         type => 'string',
1920         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1921         format_description => 'UUID',
1922         description => "Set SMBIOS1 UUID.",
1923         optional => 1,
1924     },
1925     version => {
1926         type => 'string',
1927         pattern => '\S+',
1928         format_description => 'string',
1929         description => "Set SMBIOS1 version.",
1930         optional => 1,
1931     },
1932     serial => {
1933         type => 'string',
1934         pattern => '\S+',
1935         format_description => 'string',
1936         description => "Set SMBIOS1 serial number.",
1937         optional => 1,
1938     },
1939     manufacturer => {
1940         type => 'string',
1941         pattern => '\S+',
1942         format_description => 'string',
1943         description => "Set SMBIOS1 manufacturer.",
1944         optional => 1,
1945     },
1946     product => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 product ID.",
1951         optional => 1,
1952     },
1953     sku => {
1954         type => 'string',
1955         pattern => '\S+',
1956         format_description => 'string',
1957         description => "Set SMBIOS1 SKU string.",
1958         optional => 1,
1959     },
1960     family => {
1961         type => 'string',
1962         pattern => '\S+',
1963         format_description => 'string',
1964         description => "Set SMBIOS1 family string.",
1965         optional => 1,
1966     },
1967 };
1968
1969 sub parse_smbios1 {
1970     my ($data) = @_;
1971
1972     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1973     warn $@ if $@;
1974     return $res;
1975 }
1976
1977 sub print_smbios1 {
1978     my ($smbios1) = @_;
1979     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1980 }
1981
1982 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1983
1984 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1985 sub verify_bootdisk {
1986     my ($value, $noerr) = @_;
1987
1988     return $value if is_valid_drivename($value);
1989
1990     return undef if $noerr;
1991
1992     die "invalid boot disk '$value'\n";
1993 }
1994
1995 sub parse_watchdog {
1996     my ($value) = @_;
1997
1998     return undef if !$value;
1999
2000     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2001     warn $@ if $@;
2002     return $res;
2003 }
2004
2005 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2006 sub verify_usb_device {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if parse_usb_device($value);
2010
2011     return undef if $noerr;
2012
2013     die "unable to parse usb device\n";
2014 }
2015
2016 # add JSON properties for create and set function
2017 sub json_config_properties {
2018     my $prop = shift;
2019
2020     foreach my $opt (keys %$confdesc) {
2021         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2022         $prop->{$opt} = $confdesc->{$opt};
2023     }
2024
2025     return $prop;
2026 }
2027
2028 sub check_type {
2029     my ($key, $value) = @_;
2030
2031     die "unknown setting '$key'\n" if !$confdesc->{$key};
2032
2033     my $type = $confdesc->{$key}->{type};
2034
2035     if (!defined($value)) {
2036         die "got undefined value\n";
2037     }
2038
2039     if ($value =~ m/[\n\r]/) {
2040         die "property contains a line feed\n";
2041     }
2042
2043     if ($type eq 'boolean') {
2044         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2045         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2046         die "type check ('boolean') failed - got '$value'\n";
2047     } elsif ($type eq 'integer') {
2048         return int($1) if $value =~ m/^(\d+)$/;
2049         die "type check ('integer') failed - got '$value'\n";
2050     } elsif ($type eq 'number') {
2051         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2052         die "type check ('number') failed - got '$value'\n";
2053     } elsif ($type eq 'string') {
2054         if (my $fmt = $confdesc->{$key}->{format}) {
2055             PVE::JSONSchema::check_format($fmt, $value);
2056             return $value;
2057         }
2058         $value =~ s/^\"(.*)\"$/$1/;
2059         return $value;
2060     } else {
2061         die "internal error"
2062     }
2063 }
2064
2065 sub check_iommu_support{
2066     #fixme : need to check IOMMU support
2067     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2068
2069     my $iommu=1;
2070     return $iommu;
2071
2072 }
2073
2074 sub touch_config {
2075     my ($vmid) = @_;
2076
2077     my $conf = PVE::QemuConfig->config_file($vmid);
2078     utime undef, undef, $conf;
2079 }
2080
2081 sub destroy_vm {
2082     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2083
2084     my $conffile = PVE::QemuConfig->config_file($vmid);
2085
2086     my $conf = PVE::QemuConfig->load_config($vmid);
2087
2088     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2089
2090     # only remove disks owned by this VM
2091     foreach_drive($conf, sub {
2092         my ($ds, $drive) = @_;
2093
2094         return if drive_is_cdrom($drive);
2095
2096         my $volid = $drive->{file};
2097
2098         return if !$volid || $volid =~ m|^/|;
2099
2100         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2101         return if !$path || !$owner || ($owner != $vmid);
2102
2103         eval {
2104             PVE::Storage::vdisk_free($storecfg, $volid);
2105         };
2106         warn "Could not remove disk '$volid', check manually: $@" if $@;
2107
2108     });
2109
2110     if ($keep_empty_config) {
2111         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2112     } else {
2113         unlink $conffile;
2114     }
2115
2116     # also remove unused disk
2117     eval {
2118         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2119
2120         eval {
2121             PVE::Storage::foreach_volid($dl, sub {
2122                 my ($volid, $sid, $volname, $d) = @_;
2123                 PVE::Storage::vdisk_free($storecfg, $volid);
2124             });
2125         };
2126         warn $@ if $@;
2127
2128     };
2129     warn $@ if $@;
2130 }
2131
2132 sub parse_vm_config {
2133     my ($filename, $raw) = @_;
2134
2135     return undef if !defined($raw);
2136
2137     my $res = {
2138         digest => Digest::SHA::sha1_hex($raw),
2139         snapshots => {},
2140         pending => {},
2141     };
2142
2143     $filename =~ m|/qemu-server/(\d+)\.conf$|
2144         || die "got strange filename '$filename'";
2145
2146     my $vmid = $1;
2147
2148     my $conf = $res;
2149     my $descr;
2150     my $section = '';
2151
2152     my @lines = split(/\n/, $raw);
2153     foreach my $line (@lines) {
2154         next if $line =~ m/^\s*$/;
2155
2156         if ($line =~ m/^\[PENDING\]\s*$/i) {
2157             $section = 'pending';
2158             if (defined($descr)) {
2159                 $descr =~ s/\s+$//;
2160                 $conf->{description} = $descr;
2161             }
2162             $descr = undef;
2163             $conf = $res->{$section} = {};
2164             next;
2165
2166         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2167             $section = $1;
2168             if (defined($descr)) {
2169                 $descr =~ s/\s+$//;
2170                 $conf->{description} = $descr;
2171             }
2172             $descr = undef;
2173             $conf = $res->{snapshots}->{$section} = {};
2174             next;
2175         }
2176
2177         if ($line =~ m/^\#(.*)\s*$/) {
2178             $descr = '' if !defined($descr);
2179             $descr .= PVE::Tools::decode_text($1) . "\n";
2180             next;
2181         }
2182
2183         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2184             $descr = '' if !defined($descr);
2185             $descr .= PVE::Tools::decode_text($2);
2186         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2187             $conf->{snapstate} = $1;
2188         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2189             my $key = $1;
2190             my $value = $2;
2191             $conf->{$key} = $value;
2192         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2193             my $value = $1;
2194             if ($section eq 'pending') {
2195                 $conf->{delete} = $value; # we parse this later
2196             } else {
2197                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2198             }
2199         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2200             my $key = $1;
2201             my $value = $2;
2202             eval { $value = check_type($key, $value); };
2203             if ($@) {
2204                 warn "vm $vmid - unable to parse value of '$key' - $@";
2205             } else {
2206                 $key = 'ide2' if $key eq 'cdrom';
2207                 my $fmt = $confdesc->{$key}->{format};
2208                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2209                     my $v = parse_drive($key, $value);
2210                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2211                         $v->{file} = $volid;
2212                         $value = print_drive($vmid, $v);
2213                     } else {
2214                         warn "vm $vmid - unable to parse value of '$key'\n";
2215                         next;
2216                     }
2217                 }
2218
2219                 $conf->{$key} = $value;
2220             }
2221         }
2222     }
2223
2224     if (defined($descr)) {
2225         $descr =~ s/\s+$//;
2226         $conf->{description} = $descr;
2227     }
2228     delete $res->{snapstate}; # just to be sure
2229
2230     return $res;
2231 }
2232
2233 sub write_vm_config {
2234     my ($filename, $conf) = @_;
2235
2236     delete $conf->{snapstate}; # just to be sure
2237
2238     if ($conf->{cdrom}) {
2239         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2240         $conf->{ide2} = $conf->{cdrom};
2241         delete $conf->{cdrom};
2242     }
2243
2244     # we do not use 'smp' any longer
2245     if ($conf->{sockets}) {
2246         delete $conf->{smp};
2247     } elsif ($conf->{smp}) {
2248         $conf->{sockets} = $conf->{smp};
2249         delete $conf->{cores};
2250         delete $conf->{smp};
2251     }
2252
2253     my $used_volids = {};
2254
2255     my $cleanup_config = sub {
2256         my ($cref, $pending, $snapname) = @_;
2257
2258         foreach my $key (keys %$cref) {
2259             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2260                 $key eq 'snapstate' || $key eq 'pending';
2261             my $value = $cref->{$key};
2262             if ($key eq 'delete') {
2263                 die "propertry 'delete' is only allowed in [PENDING]\n"
2264                     if !$pending;
2265                 # fixme: check syntax?
2266                 next;
2267             }
2268             eval { $value = check_type($key, $value); };
2269             die "unable to parse value of '$key' - $@" if $@;
2270
2271             $cref->{$key} = $value;
2272
2273             if (!$snapname && is_valid_drivename($key)) {
2274                 my $drive = parse_drive($key, $value);
2275                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2276             }
2277         }
2278     };
2279
2280     &$cleanup_config($conf);
2281
2282     &$cleanup_config($conf->{pending}, 1);
2283
2284     foreach my $snapname (keys %{$conf->{snapshots}}) {
2285         die "internal error" if $snapname eq 'pending';
2286         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2287     }
2288
2289     # remove 'unusedX' settings if we re-add a volume
2290     foreach my $key (keys %$conf) {
2291         my $value = $conf->{$key};
2292         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2293             delete $conf->{$key};
2294         }
2295     }
2296
2297     my $generate_raw_config = sub {
2298         my ($conf, $pending) = @_;
2299
2300         my $raw = '';
2301
2302         # add description as comment to top of file
2303         if (defined(my $descr = $conf->{description})) {
2304             if ($descr) {
2305                 foreach my $cl (split(/\n/, $descr)) {
2306                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2307                 }
2308             } else {
2309                 $raw .= "#\n" if $pending;
2310             }
2311         }
2312
2313         foreach my $key (sort keys %$conf) {
2314             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2315             $raw .= "$key: $conf->{$key}\n";
2316         }
2317         return $raw;
2318     };
2319
2320     my $raw = &$generate_raw_config($conf);
2321
2322     if (scalar(keys %{$conf->{pending}})){
2323         $raw .= "\n[PENDING]\n";
2324         $raw .= &$generate_raw_config($conf->{pending}, 1);
2325     }
2326
2327     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2328         $raw .= "\n[$snapname]\n";
2329         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2330     }
2331
2332     return $raw;
2333 }
2334
2335 sub load_defaults {
2336
2337     my $res = {};
2338
2339     # we use static defaults from our JSON schema configuration
2340     foreach my $key (keys %$confdesc) {
2341         if (defined(my $default = $confdesc->{$key}->{default})) {
2342             $res->{$key} = $default;
2343         }
2344     }
2345
2346     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2347     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2348
2349     return $res;
2350 }
2351
2352 sub config_list {
2353     my $vmlist = PVE::Cluster::get_vmlist();
2354     my $res = {};
2355     return $res if !$vmlist || !$vmlist->{ids};
2356     my $ids = $vmlist->{ids};
2357
2358     foreach my $vmid (keys %$ids) {
2359         my $d = $ids->{$vmid};
2360         next if !$d->{node} || $d->{node} ne $nodename;
2361         next if !$d->{type} || $d->{type} ne 'qemu';
2362         $res->{$vmid}->{exists} = 1;
2363     }
2364     return $res;
2365 }
2366
2367 # test if VM uses local resources (to prevent migration)
2368 sub check_local_resources {
2369     my ($conf, $noerr) = @_;
2370
2371     my $loc_res = 0;
2372
2373     $loc_res = 1 if $conf->{hostusb}; # old syntax
2374     $loc_res = 1 if $conf->{hostpci}; # old syntax
2375
2376     foreach my $k (keys %$conf) {
2377         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2378         # sockets are safe: they will recreated be on the target side post-migrate
2379         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2380         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2381     }
2382
2383     die "VM uses local resources\n" if $loc_res && !$noerr;
2384
2385     return $loc_res;
2386 }
2387
2388 # check if used storages are available on all nodes (use by migrate)
2389 sub check_storage_availability {
2390     my ($storecfg, $conf, $node) = @_;
2391
2392     foreach_drive($conf, sub {
2393         my ($ds, $drive) = @_;
2394
2395         my $volid = $drive->{file};
2396         return if !$volid;
2397
2398         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2399         return if !$sid;
2400
2401         # check if storage is available on both nodes
2402         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2403         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2404    });
2405 }
2406
2407 # list nodes where all VM images are available (used by has_feature API)
2408 sub shared_nodes {
2409     my ($conf, $storecfg) = @_;
2410
2411     my $nodelist = PVE::Cluster::get_nodelist();
2412     my $nodehash = { map { $_ => 1 } @$nodelist };
2413     my $nodename = PVE::INotify::nodename();
2414
2415     foreach_drive($conf, sub {
2416         my ($ds, $drive) = @_;
2417
2418         my $volid = $drive->{file};
2419         return if !$volid;
2420
2421         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2422         if ($storeid) {
2423             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2424             if ($scfg->{disable}) {
2425                 $nodehash = {};
2426             } elsif (my $avail = $scfg->{nodes}) {
2427                 foreach my $node (keys %$nodehash) {
2428                     delete $nodehash->{$node} if !$avail->{$node};
2429                 }
2430             } elsif (!$scfg->{shared}) {
2431                 foreach my $node (keys %$nodehash) {
2432                     delete $nodehash->{$node} if $node ne $nodename
2433                 }
2434             }
2435         }
2436     });
2437
2438     return $nodehash
2439 }
2440
2441 sub check_cmdline {
2442     my ($pidfile, $pid) = @_;
2443
2444     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2445     if (defined($fh)) {
2446         my $line = <$fh>;
2447         $fh->close;
2448         return undef if !$line;
2449         my @param = split(/\0/, $line);
2450
2451         my $cmd = $param[0];
2452         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2453
2454         for (my $i = 0; $i < scalar (@param); $i++) {
2455             my $p = $param[$i];
2456             next if !$p;
2457             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2458                 my $p = $param[$i+1];
2459                 return 1 if $p && ($p eq $pidfile);
2460                 return undef;
2461             }
2462         }
2463     }
2464     return undef;
2465 }
2466
2467 sub check_running {
2468     my ($vmid, $nocheck, $node) = @_;
2469
2470     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2471
2472     die "unable to find configuration file for VM $vmid - no such machine\n"
2473         if !$nocheck && ! -f $filename;
2474
2475     my $pidfile = pidfile_name($vmid);
2476
2477     if (my $fd = IO::File->new("<$pidfile")) {
2478         my $st = stat($fd);
2479         my $line = <$fd>;
2480         close($fd);
2481
2482         my $mtime = $st->mtime;
2483         if ($mtime > time()) {
2484             warn "file '$filename' modified in future\n";
2485         }
2486
2487         if ($line =~ m/^(\d+)$/) {
2488             my $pid = $1;
2489             if (check_cmdline($pidfile, $pid)) {
2490                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2491                     return $pid;
2492                 }
2493             }
2494         }
2495     }
2496
2497     return undef;
2498 }
2499
2500 sub vzlist {
2501
2502     my $vzlist = config_list();
2503
2504     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2505
2506     while (defined(my $de = $fd->read)) {
2507         next if $de !~ m/^(\d+)\.pid$/;
2508         my $vmid = $1;
2509         next if !defined($vzlist->{$vmid});
2510         if (my $pid = check_running($vmid)) {
2511             $vzlist->{$vmid}->{pid} = $pid;
2512         }
2513     }
2514
2515     return $vzlist;
2516 }
2517
2518 sub disksize {
2519     my ($storecfg, $conf) = @_;
2520
2521     my $bootdisk = $conf->{bootdisk};
2522     return undef if !$bootdisk;
2523     return undef if !is_valid_drivename($bootdisk);
2524
2525     return undef if !$conf->{$bootdisk};
2526
2527     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2528     return undef if !defined($drive);
2529
2530     return undef if drive_is_cdrom($drive);
2531
2532     my $volid = $drive->{file};
2533     return undef if !$volid;
2534
2535     return $drive->{size};
2536 }
2537
2538 my $last_proc_pid_stat;
2539
2540 # get VM status information
2541 # This must be fast and should not block ($full == false)
2542 # We only query KVM using QMP if $full == true (this can be slow)
2543 sub vmstatus {
2544     my ($opt_vmid, $full) = @_;
2545
2546     my $res = {};
2547
2548     my $storecfg = PVE::Storage::config();
2549
2550     my $list = vzlist();
2551     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2552
2553     my $cpucount = $cpuinfo->{cpus} || 1;
2554
2555     foreach my $vmid (keys %$list) {
2556         next if $opt_vmid && ($vmid ne $opt_vmid);
2557
2558         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2559         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2560
2561         my $d = {};
2562         $d->{pid} = $list->{$vmid}->{pid};
2563
2564         # fixme: better status?
2565         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2566
2567         my $size = disksize($storecfg, $conf);
2568         if (defined($size)) {
2569             $d->{disk} = 0; # no info available
2570             $d->{maxdisk} = $size;
2571         } else {
2572             $d->{disk} = 0;
2573             $d->{maxdisk} = 0;
2574         }
2575
2576         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2577         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2578         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2579
2580         $d->{name} = $conf->{name} || "VM $vmid";
2581         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2582
2583         if ($conf->{balloon}) {
2584             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2585             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2586         }
2587
2588         $d->{uptime} = 0;
2589         $d->{cpu} = 0;
2590         $d->{mem} = 0;
2591
2592         $d->{netout} = 0;
2593         $d->{netin} = 0;
2594
2595         $d->{diskread} = 0;
2596         $d->{diskwrite} = 0;
2597
2598         $d->{template} = PVE::QemuConfig->is_template($conf);
2599
2600         $res->{$vmid} = $d;
2601     }
2602
2603     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2604     foreach my $dev (keys %$netdev) {
2605         next if $dev !~ m/^tap([1-9]\d*)i/;
2606         my $vmid = $1;
2607         my $d = $res->{$vmid};
2608         next if !$d;
2609
2610         $d->{netout} += $netdev->{$dev}->{receive};
2611         $d->{netin} += $netdev->{$dev}->{transmit};
2612
2613         if ($full) {
2614             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2615             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2616         }
2617
2618     }
2619
2620     my $ctime = gettimeofday;
2621
2622     foreach my $vmid (keys %$list) {
2623
2624         my $d = $res->{$vmid};
2625         my $pid = $d->{pid};
2626         next if !$pid;
2627
2628         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2629         next if !$pstat; # not running
2630
2631         my $used = $pstat->{utime} + $pstat->{stime};
2632
2633         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2634
2635         if ($pstat->{vsize}) {
2636             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2637         }
2638
2639         my $old = $last_proc_pid_stat->{$pid};
2640         if (!$old) {
2641             $last_proc_pid_stat->{$pid} = {
2642                 time => $ctime,
2643                 used => $used,
2644                 cpu => 0,
2645             };
2646             next;
2647         }
2648
2649         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2650
2651         if ($dtime > 1000) {
2652             my $dutime = $used -  $old->{used};
2653
2654             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2655             $last_proc_pid_stat->{$pid} = {
2656                 time => $ctime,
2657                 used => $used,
2658                 cpu => $d->{cpu},
2659             };
2660         } else {
2661             $d->{cpu} = $old->{cpu};
2662         }
2663     }
2664
2665     return $res if !$full;
2666
2667     my $qmpclient = PVE::QMPClient->new();
2668
2669     my $ballooncb = sub {
2670         my ($vmid, $resp) = @_;
2671
2672         my $info = $resp->{'return'};
2673         return if !$info->{max_mem};
2674
2675         my $d = $res->{$vmid};
2676
2677         # use memory assigned to VM
2678         $d->{maxmem} = $info->{max_mem};
2679         $d->{balloon} = $info->{actual};
2680
2681         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2682             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2683             $d->{freemem} = $info->{free_mem};
2684         }
2685
2686         $d->{ballooninfo} = $info;
2687     };
2688
2689     my $blockstatscb = sub {
2690         my ($vmid, $resp) = @_;
2691         my $data = $resp->{'return'} || [];
2692         my $totalrdbytes = 0;
2693         my $totalwrbytes = 0;
2694
2695         for my $blockstat (@$data) {
2696             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2697             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2698
2699             $blockstat->{device} =~ s/drive-//;
2700             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2701         }
2702         $res->{$vmid}->{diskread} = $totalrdbytes;
2703         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2704     };
2705
2706     my $statuscb = sub {
2707         my ($vmid, $resp) = @_;
2708
2709         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2710         # this fails if ballon driver is not loaded, so this must be
2711         # the last commnand (following command are aborted if this fails).
2712         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2713
2714         my $status = 'unknown';
2715         if (!defined($status = $resp->{'return'}->{status})) {
2716             warn "unable to get VM status\n";
2717             return;
2718         }
2719
2720         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2721     };
2722
2723     foreach my $vmid (keys %$list) {
2724         next if $opt_vmid && ($vmid ne $opt_vmid);
2725         next if !$res->{$vmid}->{pid}; # not running
2726         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2727     }
2728
2729     $qmpclient->queue_execute(undef, 2);
2730
2731     foreach my $vmid (keys %$list) {
2732         next if $opt_vmid && ($vmid ne $opt_vmid);
2733         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2734     }
2735
2736     return $res;
2737 }
2738
2739 sub foreach_drive {
2740     my ($conf, $func, @param) = @_;
2741
2742     foreach my $ds (valid_drive_names()) {
2743         next if !defined($conf->{$ds});
2744
2745         my $drive = parse_drive($ds, $conf->{$ds});
2746         next if !$drive;
2747
2748         &$func($ds, $drive, @param);
2749     }
2750 }
2751
2752 sub foreach_volid {
2753     my ($conf, $func, @param) = @_;
2754
2755     my $volhash = {};
2756
2757     my $test_volid = sub {
2758         my ($volid, $is_cdrom) = @_;
2759
2760         return if !$volid;
2761
2762         $volhash->{$volid} = $is_cdrom || 0;
2763     };
2764
2765     foreach_drive($conf, sub {
2766         my ($ds, $drive) = @_;
2767         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2768     });
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         my $snap = $conf->{snapshots}->{$snapname};
2772         &$test_volid($snap->{vmstate}, 0);
2773         foreach_drive($snap, sub {
2774             my ($ds, $drive) = @_;
2775             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2776         });
2777     }
2778
2779     foreach my $volid (keys %$volhash) {
2780         &$func($volid, $volhash->{$volid}, @param);
2781     }
2782 }
2783
2784 sub vga_conf_has_spice {
2785     my ($vga) = @_;
2786
2787     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2788
2789     return $1 || 1;
2790 }
2791
2792 sub config_to_command {
2793     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2794
2795     my $cmd = [];
2796     my $globalFlags = [];
2797     my $machineFlags = [];
2798     my $rtcFlags = [];
2799     my $cpuFlags = [];
2800     my $devices = [];
2801     my $pciaddr = '';
2802     my $bridges = {};
2803     my $kvmver = kvm_user_version();
2804     my $vernum = 0; # unknown
2805     my $ostype = $conf->{ostype};
2806     my $winversion = windows_version($ostype);
2807
2808     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2809         $vernum = $1*1000000+$2*1000;
2810     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2811         $vernum = $1*1000000+$2*1000+$3;
2812     }
2813
2814     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2815
2816     my $have_ovz = -f '/proc/vz/vestat';
2817
2818     my $q35 = machine_type_is_q35($conf);
2819     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2820     my $machine_type = $forcemachine || $conf->{machine};
2821     my $use_old_bios_files = undef;
2822     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2823
2824     my $cpuunits = defined($conf->{cpuunits}) ?
2825             $conf->{cpuunits} : $defaults->{cpuunits};
2826
2827     push @$cmd, '/usr/bin/kvm';
2828
2829     push @$cmd, '-id', $vmid;
2830
2831     my $use_virtio = 0;
2832
2833     my $qmpsocket = qmp_socket($vmid);
2834     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2835     push @$cmd, '-mon', "chardev=qmp,mode=control";
2836
2837
2838     push @$cmd, '-pidfile' , pidfile_name($vmid);
2839
2840     push @$cmd, '-daemonize';
2841
2842     if ($conf->{smbios1}) {
2843         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2844     }
2845
2846     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2847         my $ovmfbase;
2848
2849         # prefer the OVMF_CODE variant
2850         if (-f $OVMF_CODE) {
2851             $ovmfbase = $OVMF_CODE;
2852         } elsif (-f $OVMF_IMG) {
2853             $ovmfbase = $OVMF_IMG;
2854         }
2855
2856         die "no uefi base img found\n" if !$ovmfbase;
2857         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2858
2859         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2860             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2861             my $format = $d->{format} // 'raw';
2862             my $path;
2863             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2864             if ($storeid) {
2865                 $path = PVE::Storage::path($storecfg, $d->{file});
2866                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2867                 $format = qemu_img_format($scfg, $volname);
2868             } else {
2869                 $path = $d->{file};
2870                 $format = "raw";
2871             }
2872             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2873         } elsif ($ovmfbase eq $OVMF_CODE) {
2874             warn "using uefi without permanent efivars disk\n";
2875             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2876             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2877             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2878         } else {
2879             # if the base img is not OVMF_CODE, we do not have to bother
2880             # to create/use a vars image, since it will not be used anyway
2881             # this can only happen if someone manually deletes the OVMF_CODE image
2882             # or has an old pve-qemu-kvm version installed.
2883             # both should not happen, but we ignore it here
2884         }
2885     }
2886
2887
2888     # add usb controllers
2889     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2890     push @$devices, @usbcontrollers if @usbcontrollers;
2891     my $vga = $conf->{vga};
2892
2893     my $qxlnum = vga_conf_has_spice($vga);
2894     $vga = 'qxl' if $qxlnum;
2895
2896     if (!$vga) {
2897         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2898     }
2899
2900     # enable absolute mouse coordinates (needed by vnc)
2901     my $tablet;
2902     if (defined($conf->{tablet})) {
2903         $tablet = $conf->{tablet};
2904     } else {
2905         $tablet = $defaults->{tablet};
2906         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2907         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2908     }
2909
2910     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2911
2912     my $kvm_off = 0;
2913     my $gpu_passthrough;
2914
2915     # host pci devices
2916     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2917         my $d = parse_hostpci($conf->{"hostpci$i"});
2918         next if !$d;
2919
2920         my $pcie = $d->{pcie};
2921         if($pcie){
2922             die "q35 machine model is not enabled" if !$q35;
2923             $pciaddr = print_pcie_addr("hostpci$i");
2924         }else{
2925             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2926         }
2927
2928         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2929         my $romfile = $d->{romfile};
2930
2931         my $xvga = '';
2932         if ($d->{'x-vga'}) {
2933             $xvga = ',x-vga=on';
2934             $kvm_off = 1;
2935             $vga = 'none';
2936             $gpu_passthrough = 1;
2937
2938             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2939                 $xvga = "";
2940             }
2941         }
2942         my $pcidevices = $d->{pciid};
2943         my $multifunction = 1 if @$pcidevices > 1;
2944
2945         my $j=0;
2946         foreach my $pcidevice (@$pcidevices) {
2947
2948             my $id = "hostpci$i";
2949             $id .= ".$j" if $multifunction;
2950             my $addr = $pciaddr;
2951             $addr .= ".$j" if $multifunction;
2952             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2953
2954             if($j == 0){
2955                 $devicestr .= "$rombar$xvga";
2956                 $devicestr .= ",multifunction=on" if $multifunction;
2957                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2958             }
2959
2960             push @$devices, '-device', $devicestr;
2961             $j++;
2962         }
2963     }
2964
2965     # usb devices
2966     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2967     push @$devices, @usbdevices if @usbdevices;
2968     # serial devices
2969     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2970         if (my $path = $conf->{"serial$i"}) {
2971             if ($path eq 'socket') {
2972                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2973                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2974                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2975             } else {
2976                 die "no such serial device\n" if ! -c $path;
2977                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2978                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2979             }
2980         }
2981     }
2982
2983     # parallel devices
2984     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2985         if (my $path = $conf->{"parallel$i"}) {
2986             die "no such parallel device\n" if ! -c $path;
2987             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2988             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2989             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2990         }
2991     }
2992
2993     my $vmname = $conf->{name} || "vm$vmid";
2994
2995     push @$cmd, '-name', $vmname;
2996
2997     my $sockets = 1;
2998     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2999     $sockets = $conf->{sockets} if  $conf->{sockets};
3000
3001     my $cores = $conf->{cores} || 1;
3002
3003     my $maxcpus = $sockets * $cores;
3004
3005     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3006
3007     my $allowed_vcpus = $cpuinfo->{cpus};
3008
3009     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3010         if ($allowed_vcpus < $maxcpus);
3011
3012     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3013
3014         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3015         for (my $i = 2; $i <= $vcpus; $i++)  {
3016             my $cpustr = print_cpu_device($conf,$i);
3017             push @$cmd, '-device', $cpustr;
3018         }
3019
3020     } else {
3021
3022         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3023     }
3024     push @$cmd, '-nodefaults';
3025
3026     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3027
3028     my $bootindex_hash = {};
3029     my $i = 1;
3030     foreach my $o (split(//, $bootorder)) {
3031         $bootindex_hash->{$o} = $i*100;
3032         $i++;
3033     }
3034
3035     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3036
3037     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3038
3039     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3040
3041     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3042
3043     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3044         my $socket = vnc_socket($vmid);
3045         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3046     } else {
3047         push @$cmd, '-nographic';
3048     }
3049
3050     # time drift fix
3051     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3052
3053     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3054     my $useLocaltime = $conf->{localtime};
3055
3056     if ($winversion >= 5) { # windows
3057         $useLocaltime = 1 if !defined($conf->{localtime});
3058
3059         # use time drift fix when acpi is enabled
3060         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3061             $tdf = 1 if !defined($conf->{tdf});
3062         }
3063     }
3064
3065     if ($winversion >= 6) {
3066         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3067         push @$cmd, '-no-hpet';
3068     }
3069
3070     push @$rtcFlags, 'driftfix=slew' if $tdf;
3071
3072     if ($nokvm) {
3073         push @$machineFlags, 'accel=tcg';
3074     } else {
3075         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3076     }
3077
3078     if ($machine_type) {
3079         push @$machineFlags, "type=${machine_type}";
3080     }
3081
3082     if ($conf->{startdate}) {
3083         push @$rtcFlags, "base=$conf->{startdate}";
3084     } elsif ($useLocaltime) {
3085         push @$rtcFlags, 'base=localtime';
3086     }
3087
3088     my $cpu = $nokvm ? "qemu64" : "kvm64";
3089     if (my $cputype = $conf->{cpu}) {
3090         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3091             or die "Cannot parse cpu description: $cputype\n";
3092         $cpu = $cpuconf->{cputype};
3093         $kvm_off = 1 if $cpuconf->{hidden};
3094     }
3095
3096     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3097
3098     push @$cpuFlags , '-x2apic'
3099         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3100
3101     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3102
3103     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3104
3105     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3106
3107         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3108         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3109     }
3110
3111     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3112
3113     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3114
3115     push @$cpuFlags, 'kvm=off' if $kvm_off;
3116
3117     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3118         die "internal error"; # should not happen
3119
3120     push @$cpuFlags, "vendor=${cpu_vendor}"
3121         if $cpu_vendor ne 'default';
3122
3123     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3124
3125     push @$cmd, '-cpu', $cpu;
3126
3127     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3128
3129     push @$cmd, '-S' if $conf->{freeze};
3130
3131     # set keyboard layout
3132     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3133     push @$cmd, '-k', $kb if $kb;
3134
3135     # enable sound
3136     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3137     #push @$cmd, '-soundhw', 'es1370';
3138     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3139
3140     if($conf->{agent}) {
3141         my $qgasocket = qmp_socket($vmid, 1);
3142         my $pciaddr = print_pci_addr("qga0", $bridges);
3143         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3144         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3145         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3146     }
3147
3148     my $spice_port;
3149
3150     if ($qxlnum) {
3151         if ($qxlnum > 1) {
3152             if ($winversion){
3153                 for(my $i = 1; $i < $qxlnum; $i++){
3154                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3155                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3156                 }
3157             } else {
3158                 # assume other OS works like Linux
3159                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3160                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3161             }
3162         }
3163
3164         my $pciaddr = print_pci_addr("spice", $bridges);
3165
3166         my $nodename = PVE::INotify::nodename();
3167         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3168         $spice_port = PVE::Tools::next_spice_port($pfamily);
3169
3170         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3171
3172         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3173         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3174         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3175     }
3176
3177     # enable balloon by default, unless explicitly disabled
3178     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3179         $pciaddr = print_pci_addr("balloon0", $bridges);
3180         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3181     }
3182
3183     if ($conf->{watchdog}) {
3184         my $wdopts = parse_watchdog($conf->{watchdog});
3185         $pciaddr = print_pci_addr("watchdog", $bridges);
3186         my $watchdog = $wdopts->{model} || 'i6300esb';
3187         push @$devices, '-device', "$watchdog$pciaddr";
3188         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3189     }
3190
3191     my $vollist = [];
3192     my $scsicontroller = {};
3193     my $ahcicontroller = {};
3194     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3195
3196     # Add iscsi initiator name if available
3197     if (my $initiator = get_initiator_name()) {
3198         push @$devices, '-iscsi', "initiator-name=$initiator";
3199     }
3200
3201     foreach_drive($conf, sub {
3202         my ($ds, $drive) = @_;
3203
3204         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3205             push @$vollist, $drive->{file};
3206         }
3207
3208         $use_virtio = 1 if $ds =~ m/^virtio/;
3209
3210         if (drive_is_cdrom ($drive)) {
3211             if ($bootindex_hash->{d}) {
3212                 $drive->{bootindex} = $bootindex_hash->{d};
3213                 $bootindex_hash->{d} += 1;
3214             }
3215         } else {
3216             if ($bootindex_hash->{c}) {
3217                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3218                 $bootindex_hash->{c} += 1;
3219             }
3220         }
3221
3222         if($drive->{interface} eq 'virtio'){
3223            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3224         }
3225
3226         if ($drive->{interface} eq 'scsi') {
3227
3228             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3229
3230             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3231             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3232
3233             my $iothread = '';
3234             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3235                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3236                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3237             } elsif ($drive->{iothread}) {
3238                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3239             }
3240
3241             my $queues = '';
3242             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3243                 $queues = ",num_queues=$drive->{queues}";
3244             }
3245
3246             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3247             $scsicontroller->{$controller}=1;
3248         }
3249
3250         if ($drive->{interface} eq 'sata') {
3251            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3252            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3253            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3254            $ahcicontroller->{$controller}=1;
3255         }
3256
3257         if ($drive->{interface} eq 'efidisk') {
3258             # this will be added somewhere else
3259             return;
3260         }
3261
3262         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3263         push @$devices, '-drive',$drive_cmd;
3264         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3265     });
3266
3267     for (my $i = 0; $i < $MAX_NETS; $i++) {
3268          next if !$conf->{"net$i"};
3269          my $d = parse_net($conf->{"net$i"});
3270          next if !$d;
3271
3272          $use_virtio = 1 if $d->{model} eq 'virtio';
3273
3274          if ($bootindex_hash->{n}) {
3275             $d->{bootindex} = $bootindex_hash->{n};
3276             $bootindex_hash->{n} += 1;
3277          }
3278
3279          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3280          push @$devices, '-netdev', $netdevfull;
3281
3282          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3283          push @$devices, '-device', $netdevicefull;
3284     }
3285
3286     if (!$q35) {
3287         # add pci bridges
3288         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3289            $bridges->{1} = 1;
3290            $bridges->{2} = 1;
3291         }
3292
3293         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3294
3295         while (my ($k, $v) = each %$bridges) {
3296             $pciaddr = print_pci_addr("pci.$k");
3297             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3298         }
3299     }
3300
3301     # add custom args
3302     if ($conf->{args}) {
3303         my $aa = PVE::Tools::split_args($conf->{args});
3304         push @$cmd, @$aa;
3305     }
3306
3307     push @$cmd, @$devices;
3308     push @$cmd, '-rtc', join(',', @$rtcFlags)
3309         if scalar(@$rtcFlags);
3310     push @$cmd, '-machine', join(',', @$machineFlags)
3311         if scalar(@$machineFlags);
3312     push @$cmd, '-global', join(',', @$globalFlags)
3313         if scalar(@$globalFlags);
3314
3315     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3316 }
3317
3318 sub vnc_socket {
3319     my ($vmid) = @_;
3320     return "${var_run_tmpdir}/$vmid.vnc";
3321 }
3322
3323 sub spice_port {
3324     my ($vmid) = @_;
3325
3326     my $res = vm_mon_cmd($vmid, 'query-spice');
3327
3328     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3329 }
3330
3331 sub qmp_socket {
3332     my ($vmid, $qga) = @_;
3333     my $sockettype = $qga ? 'qga' : 'qmp';
3334     return "${var_run_tmpdir}/$vmid.$sockettype";
3335 }
3336
3337 sub pidfile_name {
3338     my ($vmid) = @_;
3339     return "${var_run_tmpdir}/$vmid.pid";
3340 }
3341
3342 sub vm_devices_list {
3343     my ($vmid) = @_;
3344
3345     my $res = vm_mon_cmd($vmid, 'query-pci');
3346     my $devices = {};
3347     foreach my $pcibus (@$res) {
3348         foreach my $device (@{$pcibus->{devices}}) {
3349             next if !$device->{'qdev_id'};
3350             if ($device->{'pci_bridge'}) {
3351                 $devices->{$device->{'qdev_id'}} = 1;
3352                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3353                     next if !$bridge_device->{'qdev_id'};
3354                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3355                     $devices->{$device->{'qdev_id'}}++;
3356                 }
3357             } else {
3358                 $devices->{$device->{'qdev_id'}} = 1;
3359             }
3360         }
3361     }
3362
3363     my $resblock = vm_mon_cmd($vmid, 'query-block');
3364     foreach my $block (@$resblock) {
3365         if($block->{device} =~ m/^drive-(\S+)/){
3366                 $devices->{$1} = 1;
3367         }
3368     }
3369
3370     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3371     foreach my $mice (@$resmice) {
3372         if ($mice->{name} eq 'QEMU HID Tablet') {
3373             $devices->{tablet} = 1;
3374             last;
3375         }
3376     }
3377
3378     # for usb devices there is no query-usb
3379     # but we can iterate over the entries in
3380     # qom-list path=/machine/peripheral
3381     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3382     foreach my $per (@$resperipheral) {
3383         if ($per->{name} =~ m/^usb\d+$/) {
3384             $devices->{$per->{name}} = 1;
3385         }
3386     }
3387
3388     return $devices;
3389 }
3390
3391 sub vm_deviceplug {
3392     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3393
3394     my $q35 = machine_type_is_q35($conf);
3395
3396     my $devices_list = vm_devices_list($vmid);
3397     return 1 if defined($devices_list->{$deviceid});
3398
3399     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3400
3401     if ($deviceid eq 'tablet') {
3402
3403         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3404
3405     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3406
3407         die "usb hotplug currently not reliable\n";
3408         # since we can't reliably hot unplug all added usb devices
3409         # and usb passthrough disables live migration
3410         # we disable usb hotplugging for now
3411         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3412
3413     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3414
3415         qemu_iothread_add($vmid, $deviceid, $device);
3416
3417         qemu_driveadd($storecfg, $vmid, $device);
3418         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3419
3420         qemu_deviceadd($vmid, $devicefull);
3421         eval { qemu_deviceaddverify($vmid, $deviceid); };
3422         if (my $err = $@) {
3423             eval { qemu_drivedel($vmid, $deviceid); };
3424             warn $@ if $@;
3425             die $err;
3426         }
3427
3428     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3429
3430
3431         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3432         my $pciaddr = print_pci_addr($deviceid);
3433         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3434
3435         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3436
3437         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3438             qemu_iothread_add($vmid, $deviceid, $device);
3439             $devicefull .= ",iothread=iothread-$deviceid";
3440         }
3441
3442         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3443             $devicefull .= ",num_queues=$device->{queues}";
3444         }
3445
3446         qemu_deviceadd($vmid, $devicefull);
3447         qemu_deviceaddverify($vmid, $deviceid);
3448
3449     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3450
3451         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3452         qemu_driveadd($storecfg, $vmid, $device);
3453
3454         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3455         eval { qemu_deviceadd($vmid, $devicefull); };
3456         if (my $err = $@) {
3457             eval { qemu_drivedel($vmid, $deviceid); };
3458             warn $@ if $@;
3459             die $err;
3460         }
3461
3462     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3463
3464         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3465
3466         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3467         my $use_old_bios_files = undef;
3468         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3469
3470         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3471         qemu_deviceadd($vmid, $netdevicefull);
3472         eval { qemu_deviceaddverify($vmid, $deviceid); };
3473         if (my $err = $@) {
3474             eval { qemu_netdevdel($vmid, $deviceid); };
3475             warn $@ if $@;
3476             die $err;
3477         }
3478
3479     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3480
3481         my $bridgeid = $2;
3482         my $pciaddr = print_pci_addr($deviceid);
3483         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         qemu_deviceaddverify($vmid, $deviceid);
3487
3488     } else {
3489         die "can't hotplug device '$deviceid'\n";
3490     }
3491
3492     return 1;
3493 }
3494
3495 # fixme: this should raise exceptions on error!
3496 sub vm_deviceunplug {
3497     my ($vmid, $conf, $deviceid) = @_;
3498
3499     my $devices_list = vm_devices_list($vmid);
3500     return 1 if !defined($devices_list->{$deviceid});
3501
3502     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3503
3504     if ($deviceid eq 'tablet') {
3505
3506         qemu_devicedel($vmid, $deviceid);
3507
3508     } elsif ($deviceid =~ m/^usb\d+$/) {
3509
3510         die "usb hotplug currently not reliable\n";
3511         # when unplugging usb devices this way,
3512         # there may be remaining usb controllers/hubs
3513         # so we disable it for now
3514         qemu_devicedel($vmid, $deviceid);
3515         qemu_devicedelverify($vmid, $deviceid);
3516
3517     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3518
3519         qemu_devicedel($vmid, $deviceid);
3520         qemu_devicedelverify($vmid, $deviceid);
3521         qemu_drivedel($vmid, $deviceid);
3522         qemu_iothread_del($conf, $vmid, $deviceid);
3523
3524     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3525
3526         qemu_devicedel($vmid, $deviceid);
3527         qemu_devicedelverify($vmid, $deviceid);
3528         qemu_iothread_del($conf, $vmid, $deviceid);
3529
3530     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3531
3532         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3533         my $device = parse_drive($deviceid, $conf->{$deviceid});
3534         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3535
3536         qemu_devicedel($vmid, $deviceid);
3537         qemu_drivedel($vmid, $deviceid);
3538         qemu_deletescsihw($conf, $vmid, $deviceid);
3539
3540     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3541
3542         qemu_devicedel($vmid, $deviceid);
3543         qemu_devicedelverify($vmid, $deviceid);
3544         qemu_netdevdel($vmid, $deviceid);
3545
3546     } else {
3547         die "can't unplug device '$deviceid'\n";
3548     }
3549
3550     return 1;
3551 }
3552
3553 sub qemu_deviceadd {
3554     my ($vmid, $devicefull) = @_;
3555
3556     $devicefull = "driver=".$devicefull;
3557     my %options =  split(/[=,]/, $devicefull);
3558
3559     vm_mon_cmd($vmid, "device_add" , %options);
3560 }
3561
3562 sub qemu_devicedel {
3563     my ($vmid, $deviceid) = @_;
3564
3565     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3566 }
3567
3568 sub qemu_iothread_add {
3569     my($vmid, $deviceid, $device) = @_;
3570
3571     if ($device->{iothread}) {
3572         my $iothreads = vm_iothreads_list($vmid);
3573         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3574     }
3575 }
3576
3577 sub qemu_iothread_del {
3578     my($conf, $vmid, $deviceid) = @_;
3579
3580     my $device = parse_drive($deviceid, $conf->{$deviceid});
3581     if ($device->{iothread}) {
3582         my $iothreads = vm_iothreads_list($vmid);
3583         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3584     }
3585 }
3586
3587 sub qemu_objectadd {
3588     my($vmid, $objectid, $qomtype) = @_;
3589
3590     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3591
3592     return 1;
3593 }
3594
3595 sub qemu_objectdel {
3596     my($vmid, $objectid) = @_;
3597
3598     vm_mon_cmd($vmid, "object-del", id => $objectid);
3599
3600     return 1;
3601 }
3602
3603 sub qemu_driveadd {
3604     my ($storecfg, $vmid, $device) = @_;
3605
3606     my $drive = print_drive_full($storecfg, $vmid, $device);
3607     $drive =~ s/\\/\\\\/g;
3608     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3609
3610     # If the command succeeds qemu prints: "OK"
3611     return 1 if $ret =~ m/OK/s;
3612
3613     die "adding drive failed: $ret\n";
3614 }
3615
3616 sub qemu_drivedel {
3617     my($vmid, $deviceid) = @_;
3618
3619     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3620     $ret =~ s/^\s+//;
3621
3622     return 1 if $ret eq "";
3623
3624     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3625     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3626
3627     die "deleting drive $deviceid failed : $ret\n";
3628 }
3629
3630 sub qemu_deviceaddverify {
3631     my ($vmid, $deviceid) = @_;
3632
3633     for (my $i = 0; $i <= 5; $i++) {
3634          my $devices_list = vm_devices_list($vmid);
3635          return 1 if defined($devices_list->{$deviceid});
3636          sleep 1;
3637     }
3638
3639     die "error on hotplug device '$deviceid'\n";
3640 }
3641
3642
3643 sub qemu_devicedelverify {
3644     my ($vmid, $deviceid) = @_;
3645
3646     # need to verify that the device is correctly removed as device_del
3647     # is async and empty return is not reliable
3648
3649     for (my $i = 0; $i <= 5; $i++) {
3650          my $devices_list = vm_devices_list($vmid);
3651          return 1 if !defined($devices_list->{$deviceid});
3652          sleep 1;
3653     }
3654
3655     die "error on hot-unplugging device '$deviceid'\n";
3656 }
3657
3658 sub qemu_findorcreatescsihw {
3659     my ($storecfg, $conf, $vmid, $device) = @_;
3660
3661     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3662
3663     my $scsihwid="$controller_prefix$controller";
3664     my $devices_list = vm_devices_list($vmid);
3665
3666     if(!defined($devices_list->{$scsihwid})) {
3667         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3668     }
3669
3670     return 1;
3671 }
3672
3673 sub qemu_deletescsihw {
3674     my ($conf, $vmid, $opt) = @_;
3675
3676     my $device = parse_drive($opt, $conf->{$opt});
3677
3678     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3679         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3680         return 1;
3681     }
3682
3683     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3684
3685     my $devices_list = vm_devices_list($vmid);
3686     foreach my $opt (keys %{$devices_list}) {
3687         if (PVE::QemuServer::is_valid_drivename($opt)) {
3688             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3689             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3690                 return 1;
3691             }
3692         }
3693     }
3694
3695     my $scsihwid="scsihw$controller";
3696
3697     vm_deviceunplug($vmid, $conf, $scsihwid);
3698
3699     return 1;
3700 }
3701
3702 sub qemu_add_pci_bridge {
3703     my ($storecfg, $conf, $vmid, $device) = @_;
3704
3705     my $bridges = {};
3706
3707     my $bridgeid;
3708
3709     print_pci_addr($device, $bridges);
3710
3711     while (my ($k, $v) = each %$bridges) {
3712         $bridgeid = $k;
3713     }
3714     return 1 if !defined($bridgeid) || $bridgeid < 1;
3715
3716     my $bridge = "pci.$bridgeid";
3717     my $devices_list = vm_devices_list($vmid);
3718
3719     if (!defined($devices_list->{$bridge})) {
3720         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3721     }
3722
3723     return 1;
3724 }
3725
3726 sub qemu_set_link_status {
3727     my ($vmid, $device, $up) = @_;
3728
3729     vm_mon_cmd($vmid, "set_link", name => $device,
3730                up => $up ? JSON::true : JSON::false);
3731 }
3732
3733 sub qemu_netdevadd {
3734     my ($vmid, $conf, $device, $deviceid) = @_;
3735
3736     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3737     my %options =  split(/[=,]/, $netdev);
3738
3739     vm_mon_cmd($vmid, "netdev_add",  %options);
3740     return 1;
3741 }
3742
3743 sub qemu_netdevdel {
3744     my ($vmid, $deviceid) = @_;
3745
3746     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3747 }
3748
3749 sub qemu_usb_hotplug {
3750     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3751
3752     return if !$device;
3753
3754     # remove the old one first
3755     vm_deviceunplug($vmid, $conf, $deviceid);
3756
3757     # check if xhci controller is necessary and available
3758     if ($device->{usb3}) {
3759
3760         my $devicelist = vm_devices_list($vmid);
3761
3762         if (!$devicelist->{xhci}) {
3763             my $pciaddr = print_pci_addr("xhci");
3764             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3765         }
3766     }
3767     my $d = parse_usb_device($device->{host});
3768     $d->{usb3} = $device->{usb3};
3769
3770     # add the new one
3771     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3772 }
3773
3774 sub qemu_cpu_hotplug {
3775     my ($vmid, $conf, $vcpus) = @_;
3776
3777     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3778
3779     my $sockets = 1;
3780     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3781     $sockets = $conf->{sockets} if  $conf->{sockets};
3782     my $cores = $conf->{cores} || 1;
3783     my $maxcpus = $sockets * $cores;
3784
3785     $vcpus = $maxcpus if !$vcpus;
3786
3787     die "you can't add more vcpus than maxcpus\n"
3788         if $vcpus > $maxcpus;
3789
3790     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3791
3792     if ($vcpus < $currentvcpus) {
3793
3794         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3795
3796             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3797                 qemu_devicedel($vmid, "cpu$i");
3798                 my $retry = 0;
3799                 my $currentrunningvcpus = undef;
3800                 while (1) {
3801                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3802                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3803                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3804                     $retry++;
3805                     sleep 1;
3806                 }
3807                 #update conf after each succesfull cpu unplug
3808                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3809                 PVE::QemuConfig->write_config($vmid, $conf);
3810             }
3811         } else {
3812             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3813         }
3814
3815         return;
3816     }
3817
3818     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3819     die "vcpus in running vm does not match its configuration\n"
3820         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3821
3822     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3823
3824         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3825             my $cpustr = print_cpu_device($conf, $i);
3826             qemu_deviceadd($vmid, $cpustr);
3827
3828             my $retry = 0;
3829             my $currentrunningvcpus = undef;
3830             while (1) {
3831                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3832                 last if scalar(@{$currentrunningvcpus}) == $i;
3833                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3834                 sleep 1;
3835                 $retry++;
3836             }
3837             #update conf after each succesfull cpu hotplug
3838             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3839             PVE::QemuConfig->write_config($vmid, $conf);
3840         }
3841     } else {
3842
3843         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3844             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3845         }
3846     }
3847 }
3848
3849 sub qemu_block_set_io_throttle {
3850     my ($vmid, $deviceid,
3851         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3852         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3853         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3854         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3855
3856     return if !check_running($vmid) ;
3857
3858     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3859         bps => int($bps),
3860         bps_rd => int($bps_rd),
3861         bps_wr => int($bps_wr),
3862         iops => int($iops),
3863         iops_rd => int($iops_rd),
3864         iops_wr => int($iops_wr),
3865         bps_max => int($bps_max),
3866         bps_rd_max => int($bps_rd_max),
3867         bps_wr_max => int($bps_wr_max),
3868         iops_max => int($iops_max),
3869         iops_rd_max => int($iops_rd_max),
3870         iops_wr_max => int($iops_wr_max),
3871         bps_max_length => int($bps_max_length),
3872         bps_rd_max_length => int($bps_rd_max_length),
3873         bps_wr_max_length => int($bps_wr_max_length),
3874         iops_max_length => int($iops_max_length),
3875         iops_rd_max_length => int($iops_rd_max_length),
3876         iops_wr_max_length => int($iops_wr_max_length),
3877     );
3878
3879 }
3880
3881 # old code, only used to shutdown old VM after update
3882 sub __read_avail {
3883     my ($fh, $timeout) = @_;
3884
3885     my $sel = new IO::Select;
3886     $sel->add($fh);
3887
3888     my $res = '';
3889     my $buf;
3890
3891     my @ready;
3892     while (scalar (@ready = $sel->can_read($timeout))) {
3893         my $count;
3894         if ($count = $fh->sysread($buf, 8192)) {
3895             if ($buf =~ /^(.*)\(qemu\) $/s) {
3896                 $res .= $1;
3897                 last;
3898             } else {
3899                 $res .= $buf;
3900             }
3901         } else {
3902             if (!defined($count)) {
3903                 die "$!\n";
3904             }
3905             last;
3906         }
3907     }
3908
3909     die "monitor read timeout\n" if !scalar(@ready);
3910
3911     return $res;
3912 }
3913
3914 # old code, only used to shutdown old VM after update
3915 sub vm_monitor_command {
3916     my ($vmid, $cmdstr, $nocheck) = @_;
3917
3918     my $res;
3919
3920     eval {
3921         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3922
3923         my $sname = "${var_run_tmpdir}/$vmid.mon";
3924
3925         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3926             die "unable to connect to VM $vmid socket - $!\n";
3927
3928         my $timeout = 3;
3929
3930         # hack: migrate sometime blocks the monitor (when migrate_downtime
3931         # is set)
3932         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3933             $timeout = 60*60; # 1 hour
3934         }
3935
3936         # read banner;
3937         my $data = __read_avail($sock, $timeout);
3938
3939         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3940             die "got unexpected qemu monitor banner\n";
3941         }
3942
3943         my $sel = new IO::Select;
3944         $sel->add($sock);
3945
3946         if (!scalar(my @ready = $sel->can_write($timeout))) {
3947             die "monitor write error - timeout";
3948         }
3949
3950         my $fullcmd = "$cmdstr\r";
3951
3952         # syslog('info', "VM $vmid monitor command: $cmdstr");
3953
3954         my $b;
3955         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3956             die "monitor write error - $!";
3957         }
3958
3959         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3960
3961         $timeout = 20;
3962
3963         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3964             $timeout = 60*60; # 1 hour
3965         } elsif ($cmdstr =~ m/^(eject|change)/) {
3966             $timeout = 60; # note: cdrom mount command is slow
3967         }
3968         if ($res = __read_avail($sock, $timeout)) {
3969
3970             my @lines = split("\r?\n", $res);
3971
3972             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3973
3974             $res = join("\n", @lines);
3975             $res .= "\n";
3976         }
3977     };
3978
3979     my $err = $@;
3980
3981     if ($err) {
3982         syslog("err", "VM $vmid monitor command failed - $err");
3983         die $err;
3984     }
3985
3986     return $res;
3987 }
3988
3989 sub qemu_block_resize {
3990     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3991
3992     my $running = check_running($vmid);
3993
3994     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3995
3996     return if !$running;
3997
3998     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3999
4000 }
4001
4002 sub qemu_volume_snapshot {
4003     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4004
4005     my $running = check_running($vmid);
4006
4007     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4008         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4009     } else {
4010         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4011     }
4012 }
4013
4014 sub qemu_volume_snapshot_delete {
4015     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4016
4017     my $running = check_running($vmid);
4018
4019     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4020         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4021     } else {
4022         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4023     }
4024 }
4025
4026 sub set_migration_caps {
4027     my ($vmid) = @_;
4028
4029     my $cap_ref = [];
4030
4031     my $enabled_cap = {
4032         "auto-converge" => 1,
4033         "xbzrle" => 1,
4034         "x-rdma-pin-all" => 0,
4035         "zero-blocks" => 0,
4036         "compress" => 0
4037     };
4038
4039     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4040
4041     for my $supported_capability (@$supported_capabilities) {
4042         push @$cap_ref, {
4043             capability => $supported_capability->{capability},
4044             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4045         };
4046     }
4047
4048     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4049 }
4050
4051 my $fast_plug_option = {
4052     'lock' => 1,
4053     'name' => 1,
4054     'onboot' => 1,
4055     'shares' => 1,
4056     'startup' => 1,
4057     'description' => 1,
4058     'protection' => 1,
4059 };
4060
4061 # hotplug changes in [PENDING]
4062 # $selection hash can be used to only apply specified options, for
4063 # example: { cores => 1 } (only apply changed 'cores')
4064 # $errors ref is used to return error messages
4065 sub vmconfig_hotplug_pending {
4066     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4067
4068     my $defaults = load_defaults();
4069
4070     # commit values which do not have any impact on running VM first
4071     # Note: those option cannot raise errors, we we do not care about
4072     # $selection and always apply them.
4073
4074     my $add_error = sub {
4075         my ($opt, $msg) = @_;
4076         $errors->{$opt} = "hotplug problem - $msg";
4077     };
4078
4079     my $changes = 0;
4080     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4081         if ($fast_plug_option->{$opt}) {
4082             $conf->{$opt} = $conf->{pending}->{$opt};
4083             delete $conf->{pending}->{$opt};
4084             $changes = 1;
4085         }
4086     }
4087
4088     if ($changes) {
4089         PVE::QemuConfig->write_config($vmid, $conf);
4090         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4091     }
4092
4093     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4094
4095     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4096     while (my ($opt, $force) = each %$pending_delete_hash) {
4097         next if $selection && !$selection->{$opt};
4098         eval {
4099             if ($opt eq 'hotplug') {
4100                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4101             } elsif ($opt eq 'tablet') {
4102                 die "skip\n" if !$hotplug_features->{usb};
4103                 if ($defaults->{tablet}) {
4104                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4105                 } else {
4106                     vm_deviceunplug($vmid, $conf, $opt);
4107                 }
4108             } elsif ($opt =~ m/^usb\d+/) {
4109                 die "skip\n";
4110                 # since we cannot reliably hot unplug usb devices
4111                 # we are disabling it
4112                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4113                 vm_deviceunplug($vmid, $conf, $opt);
4114             } elsif ($opt eq 'vcpus') {
4115                 die "skip\n" if !$hotplug_features->{cpu};
4116                 qemu_cpu_hotplug($vmid, $conf, undef);
4117             } elsif ($opt eq 'balloon') {
4118                 # enable balloon device is not hotpluggable
4119                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4120             } elsif ($fast_plug_option->{$opt}) {
4121                 # do nothing
4122             } elsif ($opt =~ m/^net(\d+)$/) {
4123                 die "skip\n" if !$hotplug_features->{network};
4124                 vm_deviceunplug($vmid, $conf, $opt);
4125             } elsif (is_valid_drivename($opt)) {
4126                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4127                 vm_deviceunplug($vmid, $conf, $opt);
4128                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4129             } elsif ($opt =~ m/^memory$/) {
4130                 die "skip\n" if !$hotplug_features->{memory};
4131                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4132             } elsif ($opt eq 'cpuunits') {
4133                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4134             } elsif ($opt eq 'cpulimit') {
4135                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4136             } else {
4137                 die "skip\n";
4138             }
4139         };
4140         if (my $err = $@) {
4141             &$add_error($opt, $err) if $err ne "skip\n";
4142         } else {
4143             # save new config if hotplug was successful
4144             delete $conf->{$opt};
4145             vmconfig_undelete_pending_option($conf, $opt);
4146             PVE::QemuConfig->write_config($vmid, $conf);
4147             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4148         }
4149     }
4150
4151     foreach my $opt (keys %{$conf->{pending}}) {
4152         next if $selection && !$selection->{$opt};
4153         my $value = $conf->{pending}->{$opt};
4154         eval {
4155             if ($opt eq 'hotplug') {
4156                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4157             } elsif ($opt eq 'tablet') {
4158                 die "skip\n" if !$hotplug_features->{usb};
4159                 if ($value == 1) {
4160                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4161                 } elsif ($value == 0) {
4162                     vm_deviceunplug($vmid, $conf, $opt);
4163                 }
4164             } elsif ($opt =~ m/^usb\d+$/) {
4165                 die "skip\n";
4166                 # since we cannot reliably hot unplug usb devices
4167                 # we are disabling it
4168                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4169                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4170                 die "skip\n" if !$d;
4171                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4172             } elsif ($opt eq 'vcpus') {
4173                 die "skip\n" if !$hotplug_features->{cpu};
4174                 qemu_cpu_hotplug($vmid, $conf, $value);
4175             } elsif ($opt eq 'balloon') {
4176                 # enable/disable balloning device is not hotpluggable
4177                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4178                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4179                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4180
4181                 # allow manual ballooning if shares is set to zero
4182                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4183                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4184                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4185                 }
4186             } elsif ($opt =~ m/^net(\d+)$/) {
4187                 # some changes can be done without hotplug
4188                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4189                                     $vmid, $opt, $value);
4190             } elsif (is_valid_drivename($opt)) {
4191                 # some changes can be done without hotplug
4192                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4193                                      $vmid, $opt, $value, 1);
4194             } elsif ($opt =~ m/^memory$/) { #dimms
4195                 die "skip\n" if !$hotplug_features->{memory};
4196                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4197             } elsif ($opt eq 'cpuunits') {
4198                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4199             } elsif ($opt eq 'cpulimit') {
4200                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4201                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4202             } else {
4203                 die "skip\n";  # skip non-hot-pluggable options
4204             }
4205         };
4206         if (my $err = $@) {
4207             &$add_error($opt, $err) if $err ne "skip\n";
4208         } else {
4209             # save new config if hotplug was successful
4210             $conf->{$opt} = $value;
4211             delete $conf->{pending}->{$opt};
4212             PVE::QemuConfig->write_config($vmid, $conf);
4213             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4214         }
4215     }
4216 }
4217
4218 sub try_deallocate_drive {
4219     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4220
4221     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4222         my $volid = $drive->{file};
4223         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4224             my $sid = PVE::Storage::parse_volume_id($volid);
4225             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4226
4227             # check if the disk is really unused
4228             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4229                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4230             PVE::Storage::vdisk_free($storecfg, $volid);
4231             return 1;
4232         } else {
4233             # If vm is not owner of this disk remove from config
4234             return 1;
4235         }
4236     }
4237
4238     return undef;
4239 }
4240
4241 sub vmconfig_delete_or_detach_drive {
4242     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4243
4244     my $drive = parse_drive($opt, $conf->{$opt});
4245
4246     my $rpcenv = PVE::RPCEnvironment::get();
4247     my $authuser = $rpcenv->get_user();
4248
4249     if ($force) {
4250         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4251         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4252     } else {
4253         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4254     }
4255 }
4256
4257 sub vmconfig_apply_pending {
4258     my ($vmid, $conf, $storecfg) = @_;
4259
4260     # cold plug
4261
4262     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4263     while (my ($opt, $force) = each %$pending_delete_hash) {
4264         die "internal error" if $opt =~ m/^unused/;
4265         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4266         if (!defined($conf->{$opt})) {
4267             vmconfig_undelete_pending_option($conf, $opt);
4268             PVE::QemuConfig->write_config($vmid, $conf);
4269         } elsif (is_valid_drivename($opt)) {
4270             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4271             vmconfig_undelete_pending_option($conf, $opt);
4272             delete $conf->{$opt};
4273             PVE::QemuConfig->write_config($vmid, $conf);
4274         } else {
4275             vmconfig_undelete_pending_option($conf, $opt);
4276             delete $conf->{$opt};
4277             PVE::QemuConfig->write_config($vmid, $conf);
4278         }
4279     }
4280
4281     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4282
4283     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4284         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4285
4286         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4287             # skip if nothing changed
4288         } elsif (is_valid_drivename($opt)) {
4289             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4290                 if defined($conf->{$opt});
4291             $conf->{$opt} = $conf->{pending}->{$opt};
4292         } else {
4293             $conf->{$opt} = $conf->{pending}->{$opt};
4294         }
4295
4296         delete $conf->{pending}->{$opt};
4297         PVE::QemuConfig->write_config($vmid, $conf);
4298     }
4299 }
4300
4301 my $safe_num_ne = sub {
4302     my ($a, $b) = @_;
4303
4304     return 0 if !defined($a) && !defined($b);
4305     return 1 if !defined($a);
4306     return 1 if !defined($b);
4307
4308     return $a != $b;
4309 };
4310
4311 my $safe_string_ne = sub {
4312     my ($a, $b) = @_;
4313
4314     return 0 if !defined($a) && !defined($b);
4315     return 1 if !defined($a);
4316     return 1 if !defined($b);
4317
4318     return $a ne $b;
4319 };
4320
4321 sub vmconfig_update_net {
4322     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4323
4324     my $newnet = parse_net($value);
4325
4326     if ($conf->{$opt}) {
4327         my $oldnet = parse_net($conf->{$opt});
4328
4329         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4330             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4331             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4332             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4333
4334             # for non online change, we try to hot-unplug
4335             die "skip\n" if !$hotplug;
4336             vm_deviceunplug($vmid, $conf, $opt);
4337         } else {
4338
4339             die "internal error" if $opt !~ m/net(\d+)/;
4340             my $iface = "tap${vmid}i$1";
4341
4342             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4343                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4344                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4345                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4346                 PVE::Network::tap_unplug($iface);
4347                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4348             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4349                 # Rate can be applied on its own but any change above needs to
4350                 # include the rate in tap_plug since OVS resets everything.
4351                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4352             }
4353
4354             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4355                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4356             }
4357
4358             return 1;
4359         }
4360     }
4361
4362     if ($hotplug) {
4363         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4364     } else {
4365         die "skip\n";
4366     }
4367 }
4368
4369 sub vmconfig_update_disk {
4370     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4371
4372     # fixme: do we need force?
4373
4374     my $drive = parse_drive($opt, $value);
4375
4376     if ($conf->{$opt}) {
4377
4378         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4379
4380             my $media = $drive->{media} || 'disk';
4381             my $oldmedia = $old_drive->{media} || 'disk';
4382             die "unable to change media type\n" if $media ne $oldmedia;
4383
4384             if (!drive_is_cdrom($old_drive)) {
4385
4386                 if ($drive->{file} ne $old_drive->{file}) {
4387
4388                     die "skip\n" if !$hotplug;
4389
4390                     # unplug and register as unused
4391                     vm_deviceunplug($vmid, $conf, $opt);
4392                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4393
4394                 } else {
4395                     # update existing disk
4396
4397                     # skip non hotpluggable value
4398                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4399                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4400                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4401                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4402                         die "skip\n";
4403                     }
4404
4405                     # apply throttle
4406                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4407                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4408                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4409                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4410                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4411                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4412                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4413                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4414                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4415                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4416                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4417                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4418                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4419                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4420                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4421                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4422                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4423                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4424
4425                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4426                                                    ($drive->{mbps} || 0)*1024*1024,
4427                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4428                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4429                                                    $drive->{iops} || 0,
4430                                                    $drive->{iops_rd} || 0,
4431                                                    $drive->{iops_wr} || 0,
4432                                                    ($drive->{mbps_max} || 0)*1024*1024,
4433                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4434                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4435                                                    $drive->{iops_max} || 0,
4436                                                    $drive->{iops_rd_max} || 0,
4437                                                    $drive->{iops_wr_max} || 0,
4438                                                    $drive->{bps_max_length} || 1,
4439                                                    $drive->{bps_rd_max_length} || 1,
4440                                                    $drive->{bps_wr_max_length} || 1,
4441                                                    $drive->{iops_max_length} || 1,
4442                                                    $drive->{iops_rd_max_length} || 1,
4443                                                    $drive->{iops_wr_max_length} || 1);
4444
4445                     }
4446
4447                     return 1;
4448                 }
4449
4450             } else { # cdrom
4451
4452                 if ($drive->{file} eq 'none') {
4453                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4454                 } else {
4455                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4456                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4457                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4458                 }
4459
4460                 return 1;
4461             }
4462         }
4463     }
4464
4465     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4466     # hotplug new disks
4467     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4468     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4469 }
4470
4471 sub vm_start {
4472     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4473         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4474
4475     PVE::QemuConfig->lock_config($vmid, sub {
4476         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4477
4478         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4479
4480         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4481
4482         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4483
4484         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4485             vmconfig_apply_pending($vmid, $conf, $storecfg);
4486             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4487         }
4488
4489         my $defaults = load_defaults();
4490
4491         # set environment variable useful inside network script
4492         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4493
4494         my $local_volumes = {};
4495
4496         if ($targetstorage) {
4497             foreach_drive($conf, sub {
4498                 my ($ds, $drive) = @_;
4499
4500                 return if drive_is_cdrom($drive);
4501
4502                 my $volid = $drive->{file};
4503
4504                 return if !$volid;
4505
4506                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4507
4508                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4509                 return if $scfg->{shared};
4510                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4511             });
4512
4513             my $format = undef;
4514
4515             foreach my $opt (sort keys %$local_volumes) {
4516
4517                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4518                 my $drive = parse_drive($opt, $conf->{$opt});
4519
4520                 #if remote storage is specified, use default format
4521                 if ($targetstorage && $targetstorage ne "1") {
4522                     $storeid = $targetstorage;
4523                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4524                     $format = $defFormat;
4525                 } else {
4526                     #else we use same format than original
4527                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4528                     $format = qemu_img_format($scfg, $volid);
4529                 }
4530
4531                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4532                 my $newdrive = $drive;
4533                 $newdrive->{format} = $format;
4534                 $newdrive->{file} = $newvolid;
4535                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4536                 $local_volumes->{$opt} = $drivestr;
4537                 #pass drive to conf for command line
4538                 $conf->{$opt} = $drivestr;
4539             }
4540         }
4541
4542         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4543
4544         my $migrate_port = 0;
4545         my $migrate_uri;
4546         if ($statefile) {
4547             if ($statefile eq 'tcp') {
4548                 my $localip = "localhost";
4549                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4550                 my $nodename = PVE::INotify::nodename();
4551
4552                 if (!defined($migration_type)) {
4553                     if (defined($datacenterconf->{migration}->{type})) {
4554                         $migration_type = $datacenterconf->{migration}->{type};
4555                     } else {
4556                         $migration_type = 'secure';
4557                     }
4558                 }
4559
4560                 if ($migration_type eq 'insecure') {
4561                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4562                     if ($migrate_network_addr) {
4563                         $localip = $migrate_network_addr;
4564                     } else {
4565                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4566                     }
4567
4568                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4569                 }
4570
4571                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4572                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4573                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4574                 push @$cmd, '-incoming', $migrate_uri;
4575                 push @$cmd, '-S';
4576
4577             } elsif ($statefile eq 'unix') {
4578                 # should be default for secure migrations as a ssh TCP forward
4579                 # tunnel is not deterministic reliable ready and fails regurarly
4580                 # to set up in time, so use UNIX socket forwards
4581                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4582                 unlink $socket_addr;
4583
4584                 $migrate_uri = "unix:$socket_addr";
4585
4586                 push @$cmd, '-incoming', $migrate_uri;
4587                 push @$cmd, '-S';
4588
4589             } else {
4590                 push @$cmd, '-loadstate', $statefile;
4591             }
4592         } elsif ($paused) {
4593             push @$cmd, '-S';
4594         }
4595
4596         # host pci devices
4597         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4598           my $d = parse_hostpci($conf->{"hostpci$i"});
4599           next if !$d;
4600           my $pcidevices = $d->{pciid};
4601           foreach my $pcidevice (@$pcidevices) {
4602                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4603
4604                 my $info = pci_device_info("0000:$pciid");
4605                 die "IOMMU not present\n" if !check_iommu_support();
4606                 die "no pci device info for device '$pciid'\n" if !$info;
4607                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4608                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4609           }
4610         }
4611
4612         PVE::Storage::activate_volumes($storecfg, $vollist);
4613
4614         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4615             my $cmd = [];
4616             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4617             eval  { run_command($cmd); };
4618         }
4619
4620         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4621                                                   : $defaults->{cpuunits};
4622
4623         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4624
4625         my %properties = (
4626             Slice => 'qemu.slice',
4627             KillMode => 'none',
4628             CPUShares => $cpuunits
4629         );
4630
4631         if (my $cpulimit = $conf->{cpulimit}) {
4632             $properties{CPUQuota} = int($cpulimit * 100);
4633         }
4634         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4635
4636         if ($conf->{hugepages}) {
4637
4638             my $code = sub {
4639                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4640                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4641
4642                 PVE::QemuServer::Memory::hugepages_mount();
4643                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4644
4645                 eval  {
4646                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4647                     run_command($cmd, %run_params);
4648                 };
4649
4650                 if (my $err = $@) {
4651                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4652                     die $err;
4653                 }
4654
4655                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4656             };
4657             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4658
4659         } else {
4660             eval  {
4661                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4662                 run_command($cmd, %run_params);
4663             };
4664         }
4665
4666         if (my $err = $@) {
4667             # deactivate volumes if start fails
4668             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4669             die "start failed: $err";
4670         }
4671
4672         print "migration listens on $migrate_uri\n" if $migrate_uri;
4673
4674         if ($statefile && $statefile ne 'tcp')  {
4675             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4676             warn $@ if $@;
4677         }
4678
4679         #start nbd server for storage migration
4680         if ($targetstorage) {
4681             my $nodename = PVE::INotify::nodename();
4682             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4683             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4684             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4685             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4686
4687             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4688
4689             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4690
4691             foreach my $opt (sort keys %$local_volumes) {
4692                 my $volid = $local_volumes->{$opt};
4693                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4694                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4695                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4696             }
4697         }
4698
4699         if ($migratedfrom) {
4700             eval {
4701                 set_migration_caps($vmid);
4702             };
4703             warn $@ if $@;
4704
4705             if ($spice_port) {
4706                 print "spice listens on port $spice_port\n";
4707                 if ($spice_ticket) {
4708                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4709                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4710                 }
4711             }
4712
4713         } else {
4714             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4715                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4716                     if $conf->{balloon};
4717             }
4718
4719             foreach my $opt (keys %$conf) {
4720                 next if $opt !~  m/^net\d+$/;
4721                 my $nicconf = parse_net($conf->{$opt});
4722                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4723             }
4724         }
4725
4726         vm_mon_cmd_nocheck($vmid, 'qom-set',
4727                     path => "machine/peripheral/balloon0",
4728                     property => "guest-stats-polling-interval",
4729                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4730
4731     });
4732 }
4733
4734 sub vm_mon_cmd {
4735     my ($vmid, $execute, %params) = @_;
4736
4737     my $cmd = { execute => $execute, arguments => \%params };
4738     vm_qmp_command($vmid, $cmd);
4739 }
4740
4741 sub vm_mon_cmd_nocheck {
4742     my ($vmid, $execute, %params) = @_;
4743
4744     my $cmd = { execute => $execute, arguments => \%params };
4745     vm_qmp_command($vmid, $cmd, 1);
4746 }
4747
4748 sub vm_qmp_command {
4749     my ($vmid, $cmd, $nocheck) = @_;
4750
4751     my $res;
4752
4753     my $timeout;
4754     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4755         $timeout = $cmd->{arguments}->{timeout};
4756         delete $cmd->{arguments}->{timeout};
4757     }
4758
4759     eval {
4760         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4761         my $sname = qmp_socket($vmid);
4762         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4763             my $qmpclient = PVE::QMPClient->new();
4764
4765             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4766         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4767             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4768                 if scalar(%{$cmd->{arguments}});
4769             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4770         } else {
4771             die "unable to open monitor socket\n";
4772         }
4773     };
4774     if (my $err = $@) {
4775         syslog("err", "VM $vmid qmp command failed - $err");
4776         die $err;
4777     }
4778
4779     return $res;
4780 }
4781
4782 sub vm_human_monitor_command {
4783     my ($vmid, $cmdline) = @_;
4784
4785     my $res;
4786
4787     my $cmd = {
4788         execute => 'human-monitor-command',
4789         arguments => { 'command-line' => $cmdline},
4790     };
4791
4792     return vm_qmp_command($vmid, $cmd);
4793 }
4794
4795 sub vm_commandline {
4796     my ($storecfg, $vmid) = @_;
4797
4798     my $conf = PVE::QemuConfig->load_config($vmid);
4799
4800     my $defaults = load_defaults();
4801
4802     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4803
4804     return PVE::Tools::cmd2string($cmd);
4805 }
4806
4807 sub vm_reset {
4808     my ($vmid, $skiplock) = @_;
4809
4810     PVE::QemuConfig->lock_config($vmid, sub {
4811
4812         my $conf = PVE::QemuConfig->load_config($vmid);
4813
4814         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4815
4816         vm_mon_cmd($vmid, "system_reset");
4817     });
4818 }
4819
4820 sub get_vm_volumes {
4821     my ($conf) = @_;
4822
4823     my $vollist = [];
4824     foreach_volid($conf, sub {
4825         my ($volid, $is_cdrom) = @_;
4826
4827         return if $volid =~ m|^/|;
4828
4829         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4830         return if !$sid;
4831
4832         push @$vollist, $volid;
4833     });
4834
4835     return $vollist;
4836 }
4837
4838 sub vm_stop_cleanup {
4839     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4840
4841     eval {
4842
4843         if (!$keepActive) {
4844             my $vollist = get_vm_volumes($conf);
4845             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4846         }
4847
4848         foreach my $ext (qw(mon qmp pid vnc qga)) {
4849             unlink "/var/run/qemu-server/${vmid}.$ext";
4850         }
4851
4852         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4853     };
4854     warn $@ if $@; # avoid errors - just warn
4855 }
4856
4857 # Note: use $nockeck to skip tests if VM configuration file exists.
4858 # We need that when migration VMs to other nodes (files already moved)
4859 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4860 sub vm_stop {
4861     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4862
4863     $force = 1 if !defined($force) && !$shutdown;
4864
4865     if ($migratedfrom){
4866         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4867         kill 15, $pid if $pid;
4868         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4869         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4870         return;
4871     }
4872
4873     PVE::QemuConfig->lock_config($vmid, sub {
4874
4875         my $pid = check_running($vmid, $nocheck);
4876         return if !$pid;
4877
4878         my $conf;
4879         if (!$nocheck) {
4880             $conf = PVE::QemuConfig->load_config($vmid);
4881             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4882             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4883                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4884                 $timeout = $opts->{down} if $opts->{down};
4885             }
4886         }
4887
4888         $timeout = 60 if !defined($timeout);
4889
4890         eval {
4891             if ($shutdown) {
4892                 if (defined($conf) && $conf->{agent}) {
4893                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4894                 } else {
4895                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4896                 }
4897             } else {
4898                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4899             }
4900         };
4901         my $err = $@;
4902
4903         if (!$err) {
4904             my $count = 0;
4905             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4906                 $count++;
4907                 sleep 1;
4908             }
4909
4910             if ($count >= $timeout) {
4911                 if ($force) {
4912                     warn "VM still running - terminating now with SIGTERM\n";
4913                     kill 15, $pid;
4914                 } else {
4915                     die "VM quit/powerdown failed - got timeout\n";
4916                 }
4917             } else {
4918                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4919                 return;
4920             }
4921         } else {
4922             if ($force) {
4923                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4924                 kill 15, $pid;
4925             } else {
4926                 die "VM quit/powerdown failed\n";
4927             }
4928         }
4929
4930         # wait again
4931         $timeout = 10;
4932
4933         my $count = 0;
4934         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4935             $count++;
4936             sleep 1;
4937         }
4938
4939         if ($count >= $timeout) {
4940             warn "VM still running - terminating now with SIGKILL\n";
4941             kill 9, $pid;
4942             sleep 1;
4943         }
4944
4945         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4946    });
4947 }
4948
4949 sub vm_suspend {
4950     my ($vmid, $skiplock) = @_;
4951
4952     PVE::QemuConfig->lock_config($vmid, sub {
4953
4954         my $conf = PVE::QemuConfig->load_config($vmid);
4955
4956         PVE::QemuConfig->check_lock($conf)
4957             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4958
4959         vm_mon_cmd($vmid, "stop");
4960     });
4961 }
4962
4963 sub vm_resume {
4964     my ($vmid, $skiplock, $nocheck) = @_;
4965
4966     PVE::QemuConfig->lock_config($vmid, sub {
4967
4968         if (!$nocheck) {
4969
4970             my $conf = PVE::QemuConfig->load_config($vmid);
4971
4972             PVE::QemuConfig->check_lock($conf)
4973                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4974
4975             vm_mon_cmd($vmid, "cont");
4976
4977         } else {
4978             vm_mon_cmd_nocheck($vmid, "cont");
4979         }
4980     });
4981 }
4982
4983 sub vm_sendkey {
4984     my ($vmid, $skiplock, $key) = @_;
4985
4986     PVE::QemuConfig->lock_config($vmid, sub {
4987
4988         my $conf = PVE::QemuConfig->load_config($vmid);
4989
4990         # there is no qmp command, so we use the human monitor command
4991         vm_human_monitor_command($vmid, "sendkey $key");
4992     });
4993 }
4994
4995 sub vm_destroy {
4996     my ($storecfg, $vmid, $skiplock) = @_;
4997
4998     PVE::QemuConfig->lock_config($vmid, sub {
4999
5000         my $conf = PVE::QemuConfig->load_config($vmid);
5001
5002         if (!check_running($vmid)) {
5003             destroy_vm($storecfg, $vmid, undef, $skiplock);
5004         } else {
5005             die "VM $vmid is running - destroy failed\n";
5006         }
5007     });
5008 }
5009
5010 # pci helpers
5011
5012 sub file_write {
5013     my ($filename, $buf) = @_;
5014
5015     my $fh = IO::File->new($filename, "w");
5016     return undef if !$fh;
5017
5018     my $res = print $fh $buf;
5019
5020     $fh->close();
5021
5022     return $res;
5023 }
5024
5025 sub pci_device_info {
5026     my ($name) = @_;
5027
5028     my $res;
5029
5030     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5031     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5032
5033     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5034     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5035
5036     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5037     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5038
5039     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5040     return undef if !defined($product) || $product !~ s/^0x//;
5041
5042     $res = {
5043         name => $name,
5044         vendor => $vendor,
5045         product => $product,
5046         domain => $domain,
5047         bus => $bus,
5048         slot => $slot,
5049         func => $func,
5050         irq => $irq,
5051         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5052     };
5053
5054     return $res;
5055 }
5056
5057 sub pci_dev_reset {
5058     my ($dev) = @_;
5059
5060     my $name = $dev->{name};
5061
5062     my $fn = "$pcisysfs/devices/$name/reset";
5063
5064     return file_write($fn, "1");
5065 }
5066
5067 sub pci_dev_bind_to_vfio {
5068     my ($dev) = @_;
5069
5070     my $name = $dev->{name};
5071
5072     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5073
5074     if (!-d $vfio_basedir) {
5075         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5076     }
5077     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5078
5079     my $testdir = "$vfio_basedir/$name";
5080     return 1 if -d $testdir;
5081
5082     my $data = "$dev->{vendor} $dev->{product}";
5083     return undef if !file_write("$vfio_basedir/new_id", $data);
5084
5085     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5086     if (!file_write($fn, $name)) {
5087         return undef if -f $fn;
5088     }
5089
5090     $fn = "$vfio_basedir/bind";
5091     if (! -d $testdir) {
5092         return undef if !file_write($fn, $name);
5093     }
5094
5095     return -d $testdir;
5096 }
5097
5098 sub pci_dev_group_bind_to_vfio {
5099     my ($pciid) = @_;
5100
5101     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5102
5103     if (!-d $vfio_basedir) {
5104         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5105     }
5106     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5107
5108     # get IOMMU group devices
5109     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5110       my @devs = grep /^0000:/, readdir($D);
5111     closedir($D);
5112
5113     foreach my $pciid (@devs) {
5114         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5115
5116         # pci bridges, switches or root ports are not supported
5117         # they have a pci_bus subdirectory so skip them
5118         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5119
5120         my $info = pci_device_info($1);
5121         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5122     }
5123
5124     return 1;
5125 }
5126
5127 # vzdump restore implementaion
5128
5129 sub tar_archive_read_firstfile {
5130     my $archive = shift;
5131
5132     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5133
5134     # try to detect archive type first
5135     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5136         die "unable to open file '$archive'\n";
5137     my $firstfile = <$fh>;
5138     kill 15, $pid;
5139     close $fh;
5140
5141     die "ERROR: archive contaions no data\n" if !$firstfile;
5142     chomp $firstfile;
5143
5144     return $firstfile;
5145 }
5146
5147 sub tar_restore_cleanup {
5148     my ($storecfg, $statfile) = @_;
5149
5150     print STDERR "starting cleanup\n";
5151
5152     if (my $fd = IO::File->new($statfile, "r")) {
5153         while (defined(my $line = <$fd>)) {
5154             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5155                 my $volid = $2;
5156                 eval {
5157                     if ($volid =~ m|^/|) {
5158                         unlink $volid || die 'unlink failed\n';
5159                     } else {
5160                         PVE::Storage::vdisk_free($storecfg, $volid);
5161                     }
5162                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5163                 };
5164                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5165             } else {
5166                 print STDERR "unable to parse line in statfile - $line";
5167             }
5168         }
5169         $fd->close();
5170     }
5171 }
5172
5173 sub restore_archive {
5174     my ($archive, $vmid, $user, $opts) = @_;
5175
5176     my $format = $opts->{format};
5177     my $comp;
5178
5179     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5180         $format = 'tar' if !$format;
5181         $comp = 'gzip';
5182     } elsif ($archive =~ m/\.tar$/) {
5183         $format = 'tar' if !$format;
5184     } elsif ($archive =~ m/.tar.lzo$/) {
5185         $format = 'tar' if !$format;
5186         $comp = 'lzop';
5187     } elsif ($archive =~ m/\.vma$/) {
5188         $format = 'vma' if !$format;
5189     } elsif ($archive =~ m/\.vma\.gz$/) {
5190         $format = 'vma' if !$format;
5191         $comp = 'gzip';
5192     } elsif ($archive =~ m/\.vma\.lzo$/) {
5193         $format = 'vma' if !$format;
5194         $comp = 'lzop';
5195     } else {
5196         $format = 'vma' if !$format; # default
5197     }
5198
5199     # try to detect archive format
5200     if ($format eq 'tar') {
5201         return restore_tar_archive($archive, $vmid, $user, $opts);
5202     } else {
5203         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5204     }
5205 }
5206
5207 sub restore_update_config_line {
5208     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5209
5210     return if $line =~ m/^\#qmdump\#/;
5211     return if $line =~ m/^\#vzdump\#/;
5212     return if $line =~ m/^lock:/;
5213     return if $line =~ m/^unused\d+:/;
5214     return if $line =~ m/^parent:/;
5215     return if $line =~ m/^template:/; # restored VM is never a template
5216
5217     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5218     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5219         # try to convert old 1.X settings
5220         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5221         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5222             my ($model, $macaddr) = split(/\=/, $devconfig);
5223             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5224             my $net = {
5225                 model => $model,
5226                 bridge => "vmbr$ind",
5227                 macaddr => $macaddr,
5228             };
5229             my $netstr = print_net($net);
5230
5231             print $outfd "net$cookie->{netcount}: $netstr\n";
5232             $cookie->{netcount}++;
5233         }
5234     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5235         my ($id, $netstr) = ($1, $2);
5236         my $net = parse_net($netstr);
5237         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5238         $netstr = print_net($net);
5239         print $outfd "$id: $netstr\n";
5240     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5241         my $virtdev = $1;
5242         my $value = $3;
5243         my $di = parse_drive($virtdev, $value);
5244         if (defined($di->{backup}) && !$di->{backup}) {
5245             print $outfd "#$line";
5246         } elsif ($map->{$virtdev}) {
5247             delete $di->{format}; # format can change on restore
5248             $di->{file} = $map->{$virtdev};
5249             $value = print_drive($vmid, $di);
5250             print $outfd "$virtdev: $value\n";
5251         } else {
5252             print $outfd $line;
5253         }
5254     } else {
5255         print $outfd $line;
5256     }
5257 }
5258
5259 sub scan_volids {
5260     my ($cfg, $vmid) = @_;
5261
5262     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5263
5264     my $volid_hash = {};
5265     foreach my $storeid (keys %$info) {
5266         foreach my $item (@{$info->{$storeid}}) {
5267             next if !($item->{volid} && $item->{size});
5268             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5269             $volid_hash->{$item->{volid}} = $item;
5270         }
5271     }
5272
5273     return $volid_hash;
5274 }
5275
5276 sub is_volume_in_use {
5277     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5278
5279     my $path = PVE::Storage::path($storecfg, $volid);
5280
5281     my $scan_config = sub {
5282         my ($cref, $snapname) = @_;
5283
5284         foreach my $key (keys %$cref) {
5285             my $value = $cref->{$key};
5286             if (is_valid_drivename($key)) {
5287                 next if $skip_drive && $key eq $skip_drive;
5288                 my $drive = parse_drive($key, $value);
5289                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5290                 return 1 if $volid eq $drive->{file};
5291                 if ($drive->{file} =~ m!^/!) {
5292                     return 1 if $drive->{file} eq $path;
5293                 } else {
5294                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5295                     next if !$storeid;
5296                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5297                     next if !$scfg;
5298                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5299                 }
5300             }
5301         }
5302
5303         return 0;
5304     };
5305
5306     return 1 if &$scan_config($conf);
5307
5308     undef $skip_drive;
5309
5310     foreach my $snapname (keys %{$conf->{snapshots}}) {
5311         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5312     }
5313
5314     return 0;
5315 }
5316
5317 sub update_disksize {
5318     my ($vmid, $conf, $volid_hash) = @_;
5319
5320     my $changes;
5321
5322     my $used = {};
5323
5324     # Note: it is allowed to define multiple storages with same path (alias), so
5325     # we need to check both 'volid' and real 'path' (two different volid can point
5326     # to the same path).
5327
5328     my $usedpath = {};
5329
5330     # update size info
5331     foreach my $opt (keys %$conf) {
5332         if (is_valid_drivename($opt)) {
5333             my $drive = parse_drive($opt, $conf->{$opt});
5334             my $volid = $drive->{file};
5335             next if !$volid;
5336
5337             $used->{$volid} = 1;
5338             if ($volid_hash->{$volid} &&
5339                 (my $path = $volid_hash->{$volid}->{path})) {
5340                 $usedpath->{$path} = 1;
5341             }
5342
5343             next if drive_is_cdrom($drive);
5344             next if !$volid_hash->{$volid};
5345
5346             $drive->{size} = $volid_hash->{$volid}->{size};
5347             my $new = print_drive($vmid, $drive);
5348             if ($new ne $conf->{$opt}) {
5349                 $changes = 1;
5350                 $conf->{$opt} = $new;
5351             }
5352         }
5353     }
5354
5355     # remove 'unusedX' entry if volume is used
5356     foreach my $opt (keys %$conf) {
5357         next if $opt !~ m/^unused\d+$/;
5358         my $volid = $conf->{$opt};
5359         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5360         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5361             $changes = 1;
5362             delete $conf->{$opt};
5363         }
5364     }
5365
5366     foreach my $volid (sort keys %$volid_hash) {
5367         next if $volid =~ m/vm-$vmid-state-/;
5368         next if $used->{$volid};
5369         my $path = $volid_hash->{$volid}->{path};
5370         next if !$path; # just to be sure
5371         next if $usedpath->{$path};
5372         $changes = 1;
5373         PVE::QemuConfig->add_unused_volume($conf, $volid);
5374         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5375     }
5376
5377     return $changes;
5378 }
5379
5380 sub rescan {
5381     my ($vmid, $nolock) = @_;
5382
5383     my $cfg = PVE::Storage::config();
5384
5385     my $volid_hash = scan_volids($cfg, $vmid);
5386
5387     my $updatefn =  sub {
5388         my ($vmid) = @_;
5389
5390         my $conf = PVE::QemuConfig->load_config($vmid);
5391
5392         PVE::QemuConfig->check_lock($conf);
5393
5394         my $vm_volids = {};
5395         foreach my $volid (keys %$volid_hash) {
5396             my $info = $volid_hash->{$volid};
5397             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5398         }
5399
5400         my $changes = update_disksize($vmid, $conf, $vm_volids);
5401
5402         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5403     };
5404
5405     if (defined($vmid)) {
5406         if ($nolock) {
5407             &$updatefn($vmid);
5408         } else {
5409             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5410         }
5411     } else {
5412         my $vmlist = config_list();
5413         foreach my $vmid (keys %$vmlist) {
5414             if ($nolock) {
5415                 &$updatefn($vmid);
5416             } else {
5417                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5418             }
5419         }
5420     }
5421 }
5422
5423 sub restore_vma_archive {
5424     my ($archive, $vmid, $user, $opts, $comp) = @_;
5425
5426     my $input = $archive eq '-' ? "<&STDIN" : undef;
5427     my $readfrom = $archive;
5428
5429     my $uncomp = '';
5430     if ($comp) {
5431         $readfrom = '-';
5432         my $qarchive = PVE::Tools::shellquote($archive);
5433         if ($comp eq 'gzip') {
5434             $uncomp = "zcat $qarchive|";
5435         } elsif ($comp eq 'lzop') {
5436             $uncomp = "lzop -d -c $qarchive|";
5437         } else {
5438             die "unknown compression method '$comp'\n";
5439         }
5440
5441     }
5442
5443     my $tmpdir = "/var/tmp/vzdumptmp$$";
5444     rmtree $tmpdir;
5445
5446     # disable interrupts (always do cleanups)
5447     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5448         warn "got interrupt - ignored\n";
5449     };
5450
5451     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5452     POSIX::mkfifo($mapfifo, 0600);
5453     my $fifofh;
5454
5455     my $openfifo = sub {
5456         open($fifofh, '>', $mapfifo) || die $!;
5457     };
5458
5459     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5460
5461     my $oldtimeout;
5462     my $timeout = 5;
5463
5464     my $devinfo = {};
5465
5466     my $rpcenv = PVE::RPCEnvironment::get();
5467
5468     my $conffile = PVE::QemuConfig->config_file($vmid);
5469     my $tmpfn = "$conffile.$$.tmp";
5470
5471     # Note: $oldconf is undef if VM does not exists
5472     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5473     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5474
5475     my $print_devmap = sub {
5476         my $virtdev_hash = {};
5477
5478         my $cfgfn = "$tmpdir/qemu-server.conf";
5479
5480         # we can read the config - that is already extracted
5481         my $fh = IO::File->new($cfgfn, "r") ||
5482             "unable to read qemu-server.conf - $!\n";
5483
5484         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5485         if (-f $fwcfgfn) {
5486             my $pve_firewall_dir = '/etc/pve/firewall';
5487             mkdir $pve_firewall_dir; # make sure the dir exists
5488             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5489         }
5490
5491         while (defined(my $line = <$fh>)) {
5492             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5493                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5494                 die "archive does not contain data for drive '$virtdev'\n"
5495                     if !$devinfo->{$devname};
5496                 if (defined($opts->{storage})) {
5497                     $storeid = $opts->{storage} || 'local';
5498                 } elsif (!$storeid) {
5499                     $storeid = 'local';
5500                 }
5501                 $format = 'raw' if !$format;
5502                 $devinfo->{$devname}->{devname} = $devname;
5503                 $devinfo->{$devname}->{virtdev} = $virtdev;
5504                 $devinfo->{$devname}->{format} = $format;
5505                 $devinfo->{$devname}->{storeid} = $storeid;
5506
5507                 # check permission on storage
5508                 my $pool = $opts->{pool}; # todo: do we need that?
5509                 if ($user ne 'root@pam') {
5510                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5511                 }
5512
5513                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5514             }
5515         }
5516
5517         foreach my $devname (keys %$devinfo) {
5518             die "found no device mapping information for device '$devname'\n"
5519                 if !$devinfo->{$devname}->{virtdev};
5520         }
5521
5522         my $cfg = PVE::Storage::config();
5523
5524         # create empty/temp config
5525         if ($oldconf) {
5526             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5527             foreach_drive($oldconf, sub {
5528                 my ($ds, $drive) = @_;
5529
5530                 return if drive_is_cdrom($drive);
5531
5532                 my $volid = $drive->{file};
5533
5534                 return if !$volid || $volid =~ m|^/|;
5535
5536                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5537                 return if !$path || !$owner || ($owner != $vmid);
5538
5539                 # Note: only delete disk we want to restore
5540                 # other volumes will become unused
5541                 if ($virtdev_hash->{$ds}) {
5542                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5543                     if (my $err = $@) {
5544                         warn $err;
5545                     }
5546                 }
5547             });
5548
5549             # delete vmstate files
5550             # since after the restore we have no snapshots anymore
5551             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5552                 my $snap = $oldconf->{snapshots}->{$snapname};
5553                 if ($snap->{vmstate}) {
5554                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5555                     if (my $err = $@) {
5556                         warn $err;
5557                     }
5558                 }
5559             }
5560         }
5561
5562         my $map = {};
5563         foreach my $virtdev (sort keys %$virtdev_hash) {
5564             my $d = $virtdev_hash->{$virtdev};
5565             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5566             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5567
5568             # test if requested format is supported
5569             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5570             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5571             $d->{format} = $defFormat if !$supported;
5572
5573             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5574                                                   $d->{format}, undef, $alloc_size);
5575             print STDERR "new volume ID is '$volid'\n";
5576             $d->{volid} = $volid;
5577             my $path = PVE::Storage::path($cfg, $volid);
5578
5579             PVE::Storage::activate_volumes($cfg,[$volid]);
5580
5581             my $write_zeros = 1;
5582             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5583                 $write_zeros = 0;
5584             }
5585
5586             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5587
5588             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5589             $map->{$virtdev} = $volid;
5590         }
5591
5592         $fh->seek(0, 0) || die "seek failed - $!\n";
5593
5594         my $outfd = new IO::File ($tmpfn, "w") ||
5595             die "unable to write config for VM $vmid\n";
5596
5597         my $cookie = { netcount => 0 };
5598         while (defined(my $line = <$fh>)) {
5599             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5600         }
5601
5602         $fh->close();
5603         $outfd->close();
5604     };
5605
5606     eval {
5607         # enable interrupts
5608         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5609             die "interrupted by signal\n";
5610         };
5611         local $SIG{ALRM} = sub { die "got timeout\n"; };
5612
5613         $oldtimeout = alarm($timeout);
5614
5615         my $parser = sub {
5616             my $line = shift;
5617
5618             print "$line\n";
5619
5620             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5621                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5622                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5623             } elsif ($line =~ m/^CTIME: /) {
5624                 # we correctly received the vma config, so we can disable
5625                 # the timeout now for disk allocation (set to 10 minutes, so
5626                 # that we always timeout if something goes wrong)
5627                 alarm(600);
5628                 &$print_devmap();
5629                 print $fifofh "done\n";
5630                 my $tmp = $oldtimeout || 0;
5631                 $oldtimeout = undef;
5632                 alarm($tmp);
5633                 close($fifofh);
5634             }
5635         };
5636
5637         print "restore vma archive: $cmd\n";
5638         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5639     };
5640     my $err = $@;
5641
5642     alarm($oldtimeout) if $oldtimeout;
5643
5644     my $vollist = [];
5645     foreach my $devname (keys %$devinfo) {
5646         my $volid = $devinfo->{$devname}->{volid};
5647         push @$vollist, $volid if $volid;
5648     }
5649
5650     my $cfg = PVE::Storage::config();
5651     PVE::Storage::deactivate_volumes($cfg, $vollist);
5652
5653     unlink $mapfifo;
5654
5655     if ($err) {
5656         rmtree $tmpdir;
5657         unlink $tmpfn;
5658
5659         foreach my $devname (keys %$devinfo) {
5660             my $volid = $devinfo->{$devname}->{volid};
5661             next if !$volid;
5662             eval {
5663                 if ($volid =~ m|^/|) {
5664                     unlink $volid || die 'unlink failed\n';
5665                 } else {
5666                     PVE::Storage::vdisk_free($cfg, $volid);
5667                 }
5668                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5669             };
5670             print STDERR "unable to cleanup '$volid' - $@" if $@;
5671         }
5672         die $err;
5673     }
5674
5675     rmtree $tmpdir;
5676
5677     rename($tmpfn, $conffile) ||
5678         die "unable to commit configuration file '$conffile'\n";
5679
5680     PVE::Cluster::cfs_update(); # make sure we read new file
5681
5682     eval { rescan($vmid, 1); };
5683     warn $@ if $@;
5684 }
5685
5686 sub restore_tar_archive {
5687     my ($archive, $vmid, $user, $opts) = @_;
5688
5689     if ($archive ne '-') {
5690         my $firstfile = tar_archive_read_firstfile($archive);
5691         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5692             if $firstfile ne 'qemu-server.conf';
5693     }
5694
5695     my $storecfg = PVE::Storage::config();
5696
5697     # destroy existing data - keep empty config
5698     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5699     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5700
5701     my $tocmd = "/usr/lib/qemu-server/qmextract";
5702
5703     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5704     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5705     $tocmd .= ' --prealloc' if $opts->{prealloc};
5706     $tocmd .= ' --info' if $opts->{info};
5707
5708     # tar option "xf" does not autodetect compression when read from STDIN,
5709     # so we pipe to zcat
5710     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5711         PVE::Tools::shellquote("--to-command=$tocmd");
5712
5713     my $tmpdir = "/var/tmp/vzdumptmp$$";
5714     mkpath $tmpdir;
5715
5716     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5717     local $ENV{VZDUMP_VMID} = $vmid;
5718     local $ENV{VZDUMP_USER} = $user;
5719
5720     my $conffile = PVE::QemuConfig->config_file($vmid);
5721     my $tmpfn = "$conffile.$$.tmp";
5722
5723     # disable interrupts (always do cleanups)
5724     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5725         print STDERR "got interrupt - ignored\n";
5726     };
5727
5728     eval {
5729         # enable interrupts
5730         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5731             die "interrupted by signal\n";
5732         };
5733
5734         if ($archive eq '-') {
5735             print "extracting archive from STDIN\n";
5736             run_command($cmd, input => "<&STDIN");
5737         } else {
5738             print "extracting archive '$archive'\n";
5739             run_command($cmd);
5740         }
5741
5742         return if $opts->{info};
5743
5744         # read new mapping
5745         my $map = {};
5746         my $statfile = "$tmpdir/qmrestore.stat";
5747         if (my $fd = IO::File->new($statfile, "r")) {
5748             while (defined (my $line = <$fd>)) {
5749                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5750                     $map->{$1} = $2 if $1;
5751                 } else {
5752                     print STDERR "unable to parse line in statfile - $line\n";
5753                 }
5754             }
5755             $fd->close();
5756         }
5757
5758         my $confsrc = "$tmpdir/qemu-server.conf";
5759
5760         my $srcfd = new IO::File($confsrc, "r") ||
5761             die "unable to open file '$confsrc'\n";
5762
5763         my $outfd = new IO::File ($tmpfn, "w") ||
5764             die "unable to write config for VM $vmid\n";
5765
5766         my $cookie = { netcount => 0 };
5767         while (defined (my $line = <$srcfd>)) {
5768             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5769         }
5770
5771         $srcfd->close();
5772         $outfd->close();
5773     };
5774     my $err = $@;
5775
5776     if ($err) {
5777
5778         unlink $tmpfn;
5779
5780         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5781
5782         die $err;
5783     }
5784
5785     rmtree $tmpdir;
5786
5787     rename $tmpfn, $conffile ||
5788         die "unable to commit configuration file '$conffile'\n";
5789
5790     PVE::Cluster::cfs_update(); # make sure we read new file
5791
5792     eval { rescan($vmid, 1); };
5793     warn $@ if $@;
5794 };
5795
5796 sub foreach_writable_storage {
5797     my ($conf, $func) = @_;
5798
5799     my $sidhash = {};
5800
5801     foreach my $ds (keys %$conf) {
5802         next if !is_valid_drivename($ds);
5803
5804         my $drive = parse_drive($ds, $conf->{$ds});
5805         next if !$drive;
5806         next if drive_is_cdrom($drive);
5807
5808         my $volid = $drive->{file};
5809
5810         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5811         $sidhash->{$sid} = $sid if $sid;
5812     }
5813
5814     foreach my $sid (sort keys %$sidhash) {
5815         &$func($sid);
5816     }
5817 }
5818
5819 sub do_snapshots_with_qemu {
5820     my ($storecfg, $volid) = @_;
5821
5822     my $storage_name = PVE::Storage::parse_volume_id($volid);
5823
5824     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5825         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5826         return 1;
5827     }
5828
5829     if ($volid =~ m/\.(qcow2|qed)$/){
5830         return 1;
5831     }
5832
5833     return undef;
5834 }
5835
5836 sub qga_check_running {
5837     my ($vmid) = @_;
5838
5839     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5840     if ($@) {
5841         warn "Qemu Guest Agent are not running - $@";
5842         return 0;
5843     }
5844     return 1;
5845 }
5846
5847 sub template_create {
5848     my ($vmid, $conf, $disk) = @_;
5849
5850     my $storecfg = PVE::Storage::config();
5851
5852     foreach_drive($conf, sub {
5853         my ($ds, $drive) = @_;
5854
5855         return if drive_is_cdrom($drive);
5856         return if $disk && $ds ne $disk;
5857
5858         my $volid = $drive->{file};
5859         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5860
5861         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5862         $drive->{file} = $voliddst;
5863         $conf->{$ds} = print_drive($vmid, $drive);
5864         PVE::QemuConfig->write_config($vmid, $conf);
5865     });
5866 }
5867
5868 sub qemu_img_convert {
5869     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5870
5871     my $storecfg = PVE::Storage::config();
5872     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5873     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5874
5875     if ($src_storeid && $dst_storeid) {
5876
5877         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5878
5879         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5880         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5881
5882         my $src_format = qemu_img_format($src_scfg, $src_volname);
5883         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5884
5885         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5886         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5887
5888         my $cmd = [];
5889         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5890         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5891         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5892         if ($is_zero_initialized) {
5893             push @$cmd, "zeroinit:$dst_path";
5894         } else {
5895             push @$cmd, $dst_path;
5896         }
5897
5898         my $parser = sub {
5899             my $line = shift;
5900             if($line =~ m/\((\S+)\/100\%\)/){
5901                 my $percent = $1;
5902                 my $transferred = int($size * $percent / 100);
5903                 my $remaining = $size - $transferred;
5904
5905                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5906             }
5907
5908         };
5909
5910         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5911         my $err = $@;
5912         die "copy failed: $err" if $err;
5913     }
5914 }
5915
5916 sub qemu_img_format {
5917     my ($scfg, $volname) = @_;
5918
5919     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5920         return $1;
5921     } else {
5922         return "raw";
5923     }
5924 }
5925
5926 sub qemu_drive_mirror {
5927     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5928
5929     $jobs = {} if !$jobs;
5930
5931     my $qemu_target;
5932     my $format;
5933     $jobs->{"drive-$drive"} = {};
5934
5935     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5936         my $server = $1;
5937         my $port = $2;
5938         my $exportname = $3;
5939
5940         $format = "nbd";
5941         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5942         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5943         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5944
5945         my $pid = fork();
5946         if (!defined($pid)) {
5947             die "forking socat tunnel failed\n";
5948         } elsif ($pid == 0) {
5949             exec(@$cmd);
5950             warn "exec failed: $!\n";
5951             POSIX::_exit(-1);
5952         }
5953         $jobs->{"drive-$drive"}->{pid} = $pid;
5954
5955         my $timeout = 0;
5956         while (!-S $unixsocket) {
5957             die "nbd connection helper timed out\n"
5958                 if $timeout++ > 5;
5959             sleep 1;
5960         }
5961     } else {
5962         my $storecfg = PVE::Storage::config();
5963         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5964
5965         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5966
5967         $format = qemu_img_format($dst_scfg, $dst_volname);
5968
5969         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5970
5971         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5972     }
5973
5974     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5975     $opts->{format} = $format if $format;
5976
5977     print "drive mirror is starting for drive-$drive\n";
5978
5979     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5980
5981     if (my $err = $@) {
5982         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5983         die "mirroring error: $err";
5984     }
5985
5986     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5987 }
5988
5989 sub qemu_drive_mirror_monitor {
5990     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5991
5992     eval {
5993         my $err_complete = 0;
5994
5995         while (1) {
5996             die "storage migration timed out\n" if $err_complete > 300;
5997
5998             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5999
6000             my $running_mirror_jobs = {};
6001             foreach my $stat (@$stats) {
6002                 next if $stat->{type} ne 'mirror';
6003                 $running_mirror_jobs->{$stat->{device}} = $stat;
6004             }
6005
6006             my $readycounter = 0;
6007
6008             foreach my $job (keys %$jobs) {
6009
6010                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6011                     print "$job : finished\n";
6012                     delete $jobs->{$job};
6013                     next;
6014                 }
6015
6016                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6017
6018                 my $busy = $running_mirror_jobs->{$job}->{busy};
6019                 my $ready = $running_mirror_jobs->{$job}->{ready};
6020                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6021                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6022                     my $remaining = $total - $transferred;
6023                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6024
6025                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6026                 }
6027
6028                 $readycounter++ if $running_mirror_jobs->{$job}->{ready} eq 'true';
6029             }
6030
6031             last if scalar(keys %$jobs) == 0;
6032
6033             if ($readycounter == scalar(keys %$jobs)) {
6034                 print "all mirroring jobs are ready \n";
6035                 last if $skipcomplete; #do the complete later
6036
6037                 if ($vmiddst && $vmiddst != $vmid) {
6038                     if ($qga) {
6039                         print "freeze filesystem\n";
6040                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6041                     } else {
6042                         print "suspend vm\n";
6043                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6044                     }
6045
6046                     # if we clone a disk for a new target vm, we don't switch the disk
6047                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6048
6049                     if ($qga) {
6050                         print "unfreeze filesystem\n";
6051                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6052                     } else {
6053                         print "resume vm\n";
6054                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6055                     }
6056
6057                     last;
6058                 } else {
6059
6060                     foreach my $job (keys %$jobs) {
6061                         # try to switch the disk if source and destination are on the same guest
6062                         print "$job: Completing block job...\n";
6063
6064                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6065                         if ($@ =~ m/cannot be completed/) {
6066                             print "$job: Block job cannot be completed, try again.\n";
6067                             $err_complete++;
6068                         }else {
6069                             print "$job: Completed successfully.\n";
6070                             $jobs->{$job}->{complete} = 1;
6071                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6072                         }
6073                     }
6074                 }
6075             }
6076             sleep 1;
6077         }
6078     };
6079     my $err = $@;
6080
6081     if ($err) {
6082         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6083         die "mirroring error: $err";
6084     }
6085
6086 }
6087
6088 sub qemu_blockjobs_cancel {
6089     my ($vmid, $jobs) = @_;
6090
6091     foreach my $job (keys %$jobs) {
6092         print "$job: Cancelling block job\n";
6093         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6094         $jobs->{$job}->{cancel} = 1;
6095     }
6096
6097     while (1) {
6098         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6099
6100         my $running_jobs = {};
6101         foreach my $stat (@$stats) {
6102             $running_jobs->{$stat->{device}} = $stat;
6103         }
6104
6105         foreach my $job (keys %$jobs) {
6106
6107             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6108                 print "$job: Done.\n";
6109                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6110                 delete $jobs->{$job};
6111             }
6112         }
6113
6114         last if scalar(keys %$jobs) == 0;
6115
6116         sleep 1;
6117     }
6118 }
6119
6120 sub qemu_blockjobs_finish_tunnel {
6121    my ($vmid, $job, $cpid) = @_;
6122
6123    return if !$cpid;
6124
6125    for (my $i = 1; $i < 20; $i++) {
6126         my $waitpid = waitpid($cpid, WNOHANG);
6127         last if (defined($waitpid) && ($waitpid == $cpid));
6128
6129         if ($i == 10) {
6130             kill(15, $cpid);
6131         } elsif ($i >= 15) {
6132             kill(9, $cpid);
6133         }
6134         sleep (1);
6135     }
6136     unlink "/run/qemu-server/$vmid.mirror-$job";
6137 }
6138
6139 sub clone_disk {
6140     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6141         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6142
6143     my $newvolid;
6144
6145     if (!$full) {
6146         print "create linked clone of drive $drivename ($drive->{file})\n";
6147         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6148         push @$newvollist, $newvolid;
6149     } else {
6150
6151         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6152         $storeid = $storage if $storage;
6153
6154         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6155         if (!$format) {
6156             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6157             $format = qemu_img_format($scfg, $volname);
6158         }
6159
6160         # test if requested format is supported - else use default
6161         my $supported = grep { $_ eq $format } @$validFormats;
6162         $format = $defFormat if !$supported;
6163
6164         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6165
6166         print "create full clone of drive $drivename ($drive->{file})\n";
6167         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6168         push @$newvollist, $newvolid;
6169
6170         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6171
6172         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6173         if (!$running || $snapname) {
6174             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6175         } else {
6176
6177             my $kvmver = get_running_qemu_version ($vmid);
6178             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6179                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6180                     if $drive->{iothread};
6181             }
6182
6183             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6184         }
6185     }
6186
6187     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6188
6189     my $disk = $drive;
6190     $disk->{format} = undef;
6191     $disk->{file} = $newvolid;
6192     $disk->{size} = $size;
6193
6194     return $disk;
6195 }
6196
6197 # this only works if VM is running
6198 sub get_current_qemu_machine {
6199     my ($vmid) = @_;
6200
6201     my $cmd = { execute => 'query-machines', arguments => {} };
6202     my $res = vm_qmp_command($vmid, $cmd);
6203
6204     my ($current, $default);
6205     foreach my $e (@$res) {
6206         $default = $e->{name} if $e->{'is-default'};
6207         $current = $e->{name} if $e->{'is-current'};
6208     }
6209
6210     # fallback to the default machine if current is not supported by qemu
6211     return $current || $default || 'pc';
6212 }
6213
6214 sub get_running_qemu_version {
6215     my ($vmid) = @_;
6216     my $cmd = { execute => 'query-version', arguments => {} };
6217     my $res = vm_qmp_command($vmid, $cmd);
6218     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6219 }
6220
6221 sub qemu_machine_feature_enabled {
6222     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6223
6224     my $current_major;
6225     my $current_minor;
6226
6227     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6228
6229         $current_major = $3;
6230         $current_minor = $4;
6231
6232     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6233
6234         $current_major = $1;
6235         $current_minor = $2;
6236     }
6237
6238     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6239
6240
6241 }
6242
6243 sub qemu_machine_pxe {
6244     my ($vmid, $conf, $machine) = @_;
6245
6246     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6247
6248     foreach my $opt (keys %$conf) {
6249         next if $opt !~ m/^net(\d+)$/;
6250         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6251         next if !$net;
6252         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6253         return $machine.".pxe" if $romfile =~ m/pxe/;
6254         last;
6255     }
6256
6257     return $machine;
6258 }
6259
6260 sub qemu_use_old_bios_files {
6261     my ($machine_type) = @_;
6262
6263     return if !$machine_type;
6264
6265     my $use_old_bios_files = undef;
6266
6267     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6268         $machine_type = $1;
6269         $use_old_bios_files = 1;
6270     } else {
6271         my $kvmver = kvm_user_version();
6272         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6273         # load new efi bios files on migration. So this hack is required to allow
6274         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6275         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6276         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6277     }
6278
6279     return ($use_old_bios_files, $machine_type);
6280 }
6281
6282 sub lspci {
6283
6284     my $devices = {};
6285
6286     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6287             my (undef, $id, $function) = @_;
6288             my $res = { id => $id, function => $function};
6289             push @{$devices->{$id}}, $res;
6290     });
6291
6292     return $devices;
6293 }
6294
6295 sub vm_iothreads_list {
6296     my ($vmid) = @_;
6297
6298     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6299
6300     my $iothreads = {};
6301     foreach my $iothread (@$res) {
6302         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6303     }
6304
6305     return $iothreads;
6306 }
6307
6308 sub scsihw_infos {
6309     my ($conf, $drive) = @_;
6310
6311     my $maxdev = 0;
6312
6313     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6314         $maxdev = 7;
6315     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6316         $maxdev = 1;
6317     } else {
6318         $maxdev = 256;
6319     }
6320
6321     my $controller = int($drive->{index} / $maxdev);
6322     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6323
6324     return ($maxdev, $controller, $controller_prefix);
6325 }
6326
6327 sub add_hyperv_enlighments {
6328     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6329
6330     return if $nokvm;
6331     return if $winversion < 6;
6332     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6333
6334     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6335
6336     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6337         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6338         push @$cpuFlags , 'hv_vapic';
6339         push @$cpuFlags , 'hv_time';
6340     } else {
6341         push @$cpuFlags , 'hv_spinlocks=0xffff';
6342     }
6343
6344     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6345         push @$cpuFlags , 'hv_reset';
6346         push @$cpuFlags , 'hv_vpindex';
6347         push @$cpuFlags , 'hv_runtime';
6348     }
6349
6350     if ($winversion >= 7) {
6351         push @$cpuFlags , 'hv_relaxed';
6352     }
6353 }
6354
6355 sub windows_version {
6356     my ($ostype) = @_;
6357
6358     return 0 if !$ostype;
6359
6360     my $winversion = 0;
6361
6362     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6363         $winversion = 5;
6364     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6365         $winversion = 6;
6366     } elsif ($ostype =~ m/^win(\d+)$/) {
6367         $winversion = $1;
6368     }
6369
6370     return $winversion;
6371 }
6372
6373 # bash completion helper
6374
6375 sub complete_backup_archives {
6376     my ($cmdname, $pname, $cvalue) = @_;
6377
6378     my $cfg = PVE::Storage::config();
6379
6380     my $storeid;
6381
6382     if ($cvalue =~ m/^([^:]+):/) {
6383         $storeid = $1;
6384     }
6385
6386     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6387
6388     my $res = [];
6389     foreach my $id (keys %$data) {
6390         foreach my $item (@{$data->{$id}}) {
6391             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6392             push @$res, $item->{volid} if defined($item->{volid});
6393         }
6394     }
6395
6396     return $res;
6397 }
6398
6399 my $complete_vmid_full = sub {
6400     my ($running) = @_;
6401
6402     my $idlist = vmstatus();
6403
6404     my $res = [];
6405
6406     foreach my $id (keys %$idlist) {
6407         my $d = $idlist->{$id};
6408         if (defined($running)) {
6409             next if $d->{template};
6410             next if $running && $d->{status} ne 'running';
6411             next if !$running && $d->{status} eq 'running';
6412         }
6413         push @$res, $id;
6414
6415     }
6416     return $res;
6417 };
6418
6419 sub complete_vmid {
6420     return &$complete_vmid_full();
6421 }
6422
6423 sub complete_vmid_stopped {
6424     return &$complete_vmid_full(0);
6425 }
6426
6427 sub complete_vmid_running {
6428     return &$complete_vmid_full(1);
6429 }
6430
6431 sub complete_storage {
6432
6433     my $cfg = PVE::Storage::config();
6434     my $ids = $cfg->{ids};
6435
6436     my $res = [];
6437     foreach my $sid (keys %$ids) {
6438         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6439         next if !$ids->{$sid}->{content}->{images};
6440         push @$res, $sid;
6441     }
6442
6443     return $res;
6444 }
6445
6446 sub nbd_stop {
6447     my ($vmid) = @_;
6448
6449     vm_mon_cmd($vmid, 'nbd-server-stop');
6450 }
6451
6452 1;