PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         default => 'kvm64',
 151         default_key => 1,
 152     },
 153     hidden => {
 154         description => "Do not identify as a KVM virtual machine.",
 155         type => 'boolean',
 156         optional => 1,
 157         default => 0
 158     },
 159 };
 160
 161 my $watchdog_fmt = {
 162     model => {
 163         default_key => 1,
 164         type => 'string',
 165         enum => [qw(i6300esb ib700)],
 166         description => "Watchdog type to emulate.",
 167         default => 'i6300esb',
 168         optional => 1,
 169     },
 170     action => {
 171         type => 'string',
 172         enum => [qw(reset shutdown poweroff pause debug none)],
 173         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 174         optional => 1,
 175     },
 176 };
 177 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 178
 179 my $confdesc = {
 180     onboot => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Specifies whether a VM will be started during system bootup.",
 184         default => 0,
 185     },
 186     autostart => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Automatic restart after crash (currently ignored).",
 190         default => 0,
 191     },
 192     hotplug => {
 193         optional => 1,
 194         type => 'string', format => 'pve-hotplug-features',
 195         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 196         default => 'network,disk,usb',
 197     },
 198     reboot => {
 199         optional => 1,
 200         type => 'boolean',
 201         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 202         default => 1,
 203     },
 204     lock => {
 205         optional => 1,
 206         type => 'string',
 207         description => "Lock/unlock the VM.",
 208         enum => [qw(migrate backup snapshot rollback)],
 209     },
 210     cpulimit => {
 211         optional => 1,
 212         type => 'number',
 213         description => "Limit of CPU usage.",
 214         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 215         minimum => 0,
 216         maximum => 128,
 217         default => 0,
 218     },
 219     cpuunits => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "CPU weight for a VM.",
 223         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 224         minimum => 0,
 225         maximum => 500000,
 226         default => 1024,
 227     },
 228     memory => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 232         minimum => 16,
 233         default => 512,
 234     },
 235     balloon => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 239         minimum => 0,
 240     },
 241     shares => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 245         minimum => 0,
 246         maximum => 50000,
 247         default => 1000,
 248     },
 249     keyboard => {
 250         optional => 1,
 251         type => 'string',
 252         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 253         enum => PVE::Tools::kvmkeymaplist(),
 254         default => 'en-us',
 255     },
 256     name => {
 257         optional => 1,
 258         type => 'string', format => 'dns-name',
 259         description => "Set a name for the VM. Only used on the configuration web interface.",
 260     },
 261     scsihw => {
 262         optional => 1,
 263         type => 'string',
 264         description => "SCSI controller model",
 265         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 266         default => 'lsi',
 267     },
 268     description => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 272     },
 273     ostype => {
 274         optional => 1,
 275         type => 'string',
 276         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 277         description => "Specify guest operating system.",
 278         verbose_description => <<EODESC,
 279 Specify guest operating system. This is used to enable special
 280 optimization/features for specific operating systems:
 281
 282 [horizontal]
 283 other;; unspecified OS
 284 wxp;; Microsoft Windows XP
 285 w2k;; Microsoft Windows 2000
 286 w2k3;; Microsoft Windows 2003
 287 w2k8;; Microsoft Windows 2008
 288 wvista;; Microsoft Windows Vista
 289 win7;; Microsoft Windows 7
 290 win8;; Microsoft Windows 8/2012
 291 l24;; Linux 2.4 Kernel
 292 l26;; Linux 2.6/3.X Kernel
 293 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 294 EODESC
 295     },
 296     boot => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 300         pattern => '[acdn]{1,4}',
 301         default => 'cdn',
 302     },
 303     bootdisk => {
 304         optional => 1,
 305         type => 'string', format => 'pve-qm-bootdisk',
 306         description => "Enable booting from specified disk.",
 307         pattern => '(ide|sata|scsi|virtio)\d+',
 308     },
 309     smp => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of CPUs. Please use option -sockets instead.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     sockets => {
 317         optional => 1,
 318         type => 'integer',
 319         description => "The number of CPU sockets.",
 320         minimum => 1,
 321         default => 1,
 322     },
 323     cores => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "The number of cores per socket.",
 327         minimum => 1,
 328         default => 1,
 329     },
 330     numa => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable NUMA.",
 334         default => 0,
 335     },
 336     hugepages => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Enable/disable hugepages memory.",
 340         enum => [qw(any 2 1024)],
 341     },
 342     vcpus => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Number of hotplugged vcpus.",
 346         minimum => 1,
 347         default => 0,
 348     },
 349     acpi => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable ACPI.",
 353         default => 1,
 354     },
 355     agent => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable Qemu GuestAgent.",
 359         default => 0,
 360     },
 361     kvm => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable KVM hardware virtualization.",
 365         default => 1,
 366     },
 367     tdf => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable time drift fix.",
 371         default => 0,
 372     },
 373     localtime => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 377     },
 378     freeze => {
 379         optional => 1,
 380         type => 'boolean',
 381         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 382     },
 383     vga => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Select the VGA type.",
 387         verbose_description => "Select the VGA type. If you want to use high resolution" .
 388             " modes (>= 1280x1024x16) then you should use the options " .
 389             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 390             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 391             "display sever. For win* OS you can select how many independent " .
 392             "displays you want, Linux guests can add displays them self. " .
 393             "You can also run without any graphic card, using a serial device" .
 394             " as terminal.",
 395         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 396     },
 397     watchdog => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-watchdog',
 400         description => "Create a virtual hardware watchdog device.",
 401         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 402             " (by a guest action), the watchdog must be periodically polled " .
 403             "by an agent inside the guest or else the watchdog will reset " .
 404             "the guest (or execute the respective action specified)",
 405     },
 406     startdate => {
 407         optional => 1,
 408         type => 'string',
 409         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 410         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 411         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 412         default => 'now',
 413     },
 414     startup =>  get_standard_option('pve-startup-order'),
 415     template => {
 416         optional => 1,
 417         type => 'boolean',
 418         description => "Enable/disable Template.",
 419         default => 0,
 420     },
 421     args => {
 422         optional => 1,
 423         type => 'string',
 424         description => "Arbitrary arguments passed to kvm.",
 425         verbose_description => <<EODESCR,
 426 Arbitrary arguments passed to kvm, for example:
 427
 428 args: -no-reboot -no-hpet
 429
 430 NOTE: this option is for experts only.
 431 EODESCR
 432     },
 433     tablet => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 1,
 437         description => "Enable/disable the USB tablet device.",
 438         verbose_description => "Enable/disable the USB tablet device. This device is " .
 439             "usually needed to allow absolute mouse positioning with VNC. " .
 440             "Else the mouse runs out of sync with normal VNC clients. " .
 441             "If you're running lots of console-only guests on one host, " .
 442             "you may consider disabling this to save some context switches. " .
 443             "This is turned off by default if you use spice (-vga=qxl).",
 444     },
 445     migrate_speed => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 449         minimum => 0,
 450         default => 0,
 451     },
 452     migrate_downtime => {
 453         optional => 1,
 454         type => 'number',
 455         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 456         minimum => 0,
 457         default => 0.1,
 458     },
 459     cdrom => {
 460         optional => 1,
 461         type => 'string', format => 'pve-qm-ide',
 462         typetext => '<volume>',
 463         description => "This is an alias for option -ide2",
 464     },
 465     cpu => {
 466         optional => 1,
 467         description => "Emulated CPU type.",
 468         type => 'string',
 469         format => $cpu_fmt,
 470     },
 471     parent => get_standard_option('pve-snapshot-name', {
 472         optional => 1,
 473         description => "Parent snapshot name. This is used internally, and should not be modified.",
 474     }),
 475     snaptime => {
 476         optional => 1,
 477         description => "Timestamp for snapshots.",
 478         type => 'integer',
 479         minimum => 0,
 480     },
 481     vmstate => {
 482         optional => 1,
 483         type => 'string', format => 'pve-volume-id',
 484         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 485     },
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     werror => {
 756         type => 'string',
 757         enum => [qw(enospc ignore report stop)],
 758         description => 'Write error action.',
 759         optional => 1,
 760     },
 761     aio => {
 762         type => 'string',
 763         enum => [qw(native threads)],
 764         description => 'AIO type to use.',
 765         optional => 1,
 766     },
 767     discard => {
 768         type => 'string',
 769         enum => [qw(ignore on)],
 770         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 771         optional => 1,
 772     },
 773     detect_zeroes => {
 774         type => 'boolean',
 775         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 776         optional => 1,
 777     },
 778     serial => {
 779         type => 'string',
 780         format => 'urlencoded',
 781         format_description => 'serial',
 782         maxLength => 20*3, # *3 since it's %xx url enoded
 783         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 784         optional => 1,
 785     }
 786 );
 787
 788 my %rerror_fmt = (
 789     rerror => {
 790         type => 'string',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         description => "Whether to use iothreads for this drive",
 800         optional => 1,
 801 });
 802
 803 my %model_fmt = (
 804     model => {
 805         type => 'string',
 806         format => 'urlencoded',
 807         format_description => 'model',
 808         maxLength => 40*3, # *3 since it's %xx url enoded
 809         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 810         optional => 1,
 811     },
 812 );
 813
 814 my %queues_fmt = (
 815     queues => {
 816         type => 'integer',
 817         description => "Number of queues.",
 818         minimum => 2,
 819         optional => 1
 820     }
 821 );
 822
 823 my $add_throttle_desc = sub {
 824     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 825     my $d = {
 826         type => $type,
 827         format_description => $unit,
 828         description => "Maximum $what in $longunit.",
 829         optional => 1,
 830     };
 831     $d->{minimum} = $minimum if defined($minimum);
 832     $drivedesc_base{$key} = $d;
 833 };
 834 # throughput: (leaky bucket)
 835 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 836 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 838 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 839 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 841 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 842 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 843 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 844
 845 # pools: (pool of IO before throttling starts taking effect)
 846 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 852
 853 # burst lengths
 854 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 855 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 857 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 860
 861 my $ide_fmt = {
 862     %drivedesc_base,
 863     %rerror_fmt,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879 };
 880 my $scsidesc = {
 881     optional => 1,
 882     type => 'string', format => $scsi_fmt,
 883     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 886
 887 my $sata_fmt = {
 888     %drivedesc_base,
 889     %rerror_fmt,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901     %rerror_fmt,
 902 };
 903 my $virtiodesc = {
 904     optional => 1,
 905     type => 'string', format => $virtio_fmt,
 906     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 909
 910 my $alldrive_fmt = {
 911     %drivedesc_base,
 912     %rerror_fmt,
 913     %iothread_fmt,
 914     %model_fmt,
 915     %queues_fmt,
 916 };
 917
 918 my $efidisk_fmt = {
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     format => get_standard_option('pve-qm-image-format'),
 928     size => {
 929         type => 'string',
 930         format => 'disk-size',
 931         format_description => 'DiskSize',
 932         description => "Disk size. This is purely informational and has no effect.",
 933         optional => 1,
 934     },
 935 };
 936
 937 my $efidisk_desc = {
 938     optional => 1,
 939     type => 'string', format => $efidisk_fmt,
 940     description => "Configure a Disk for storing EFI vars",
 941 };
 942
 943 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 944
 945 my $usb_fmt = {
 946     host => {
 947         default_key => 1,
 948         type => 'string', format => 'pve-qm-usb-device',
 949         format_description => 'HOSTUSBDEVICE|spice',
 950         description => <<EODESCR,
 951 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 952
 953  'bus-port(.port)*' (decimal numbers) or
 954  'vendor_id:product_id' (hexadeciaml numbers) or
 955  'spice'
 956
 957 You can use the 'lsusb -t' command to list existing usb devices.
 958
 959 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 960
 961 The value 'spice' can be used to add a usb redirection devices for spice.
 962 EODESCR
 963     },
 964     usb3 => {
 965         optional => 1,
 966         type => 'boolean',
 967         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 968         default => 0,
 969     },
 970 };
 971
 972 my $usbdesc = {
 973     optional => 1,
 974     type => 'string', format => $usb_fmt,
 975     description => "Configure an USB device (n is 0 to 4).",
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 978
 979 # NOTE: the match-groups of this regex are used in parse_hostpci
 980 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 981 my $hostpci_fmt = {
 982     host => {
 983         default_key => 1,
 984         type => 'string',
 985         pattern => qr/$PCIRE(;$PCIRE)*/,
 986         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 987         description => <<EODESCR,
 988 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 989 of PCI virtual functions of the host. HOSTPCIID syntax is:
 990
 991 'bus:dev.func' (hexadecimal numbers)
 992
 993 You can us the 'lspci' command to list existing PCI devices.
 994 EODESCR
 995     },
 996     rombar => {
 997         type => 'boolean',
 998         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 999         optional => 1,
1000         default => 1,
1001     },
1002     pcie => {
1003         type => 'boolean',
1004         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1005         optional => 1,
1006         default => 0,
1007     },
1008     'x-vga' => {
1009         type => 'boolean',
1010         description =>  "Enable vfio-vga device support.",
1011         optional => 1,
1012         default => 0,
1013     },
1014 };
1015 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1016
1017 my $hostpcidesc = {
1018         optional => 1,
1019         type => 'string', format => 'pve-qm-hostpci',
1020         description => "Map host PCI devices into guest.",
1021         verbose_description =>  <<EODESCR,
1022 Map host PCI devices into guest.
1023
1024 NOTE: This option allows direct access to host hardware. So it is no longer
1025 possible to migrate such machines - use with special care.
1026
1027 CAUTION: Experimental! User reported problems with this option.
1028 EODESCR
1029 };
1030 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1031
1032 my $serialdesc = {
1033         optional => 1,
1034         type => 'string',
1035         pattern => '(/dev/.+|socket)',
1036         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1037         verbose_description =>  <<EODESCR,
1038 Create a serial device inside the VM (n is 0 to 3), and pass through a
1039 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1040 host side (use 'qm terminal' to open a terminal connection).
1041
1042 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1043
1044 CAUTION: Experimental! User reported problems with this option.
1045 EODESCR
1046 };
1047
1048 my $paralleldesc= {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1052         description =>  "Map host parallel devices (n is 0 to 2).",
1053         verbose_description =>  <<EODESCR,
1054 Map host parallel devices (n is 0 to 2).
1055
1056 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1057
1058 CAUTION: Experimental! User reported problems with this option.
1059 EODESCR
1060 };
1061
1062 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1063     $confdesc->{"parallel$i"} = $paralleldesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1067     $confdesc->{"serial$i"} = $serialdesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1071     $confdesc->{"hostpci$i"} = $hostpcidesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1075     $drivename_hash->{"ide$i"} = 1;
1076     $confdesc->{"ide$i"} = $idedesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1080     $drivename_hash->{"sata$i"} = 1;
1081     $confdesc->{"sata$i"} = $satadesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1085     $drivename_hash->{"scsi$i"} = 1;
1086     $confdesc->{"scsi$i"} = $scsidesc ;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1090     $drivename_hash->{"virtio$i"} = 1;
1091     $confdesc->{"virtio$i"} = $virtiodesc;
1092 }
1093
1094 $drivename_hash->{efidisk0} = 1;
1095 $confdesc->{efidisk0} = $efidisk_desc;
1096
1097 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1098     $confdesc->{"usb$i"} = $usbdesc;
1099 }
1100
1101 my $unuseddesc = {
1102     optional => 1,
1103     type => 'string', format => 'pve-volume-id',
1104     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1105 };
1106
1107 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1108     $confdesc->{"unused$i"} = $unuseddesc;
1109 }
1110
1111 my $kvm_api_version = 0;
1112
1113 sub kvm_version {
1114
1115     return $kvm_api_version if $kvm_api_version;
1116
1117     my $fh = IO::File->new("</dev/kvm") ||
1118         return 0;
1119
1120     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1121         $kvm_api_version = $v;
1122     }
1123
1124     $fh->close();
1125
1126     return  $kvm_api_version;
1127 }
1128
1129 my $kvm_user_version;
1130
1131 sub kvm_user_version {
1132
1133     return $kvm_user_version if $kvm_user_version;
1134
1135     $kvm_user_version = 'unknown';
1136
1137     my $code = sub {
1138         my $line = shift;
1139         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1140             $kvm_user_version = $2;
1141         }
1142     };
1143
1144     eval { run_command("kvm -version", outfunc => $code); };
1145     warn $@ if $@;
1146
1147     return $kvm_user_version;
1148
1149 }
1150
1151 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1152
1153 sub valid_drive_names {
1154     # order is important - used to autoselect boot disk
1155     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1156             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1157             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1158             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1159             'efidisk0');
1160 }
1161
1162 sub is_valid_drivename {
1163     my $dev = shift;
1164
1165     return defined($drivename_hash->{$dev});
1166 }
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 sub nic_models {
1174     return $nic_model_list;
1175 }
1176
1177 sub os_list_description {
1178
1179     return {
1180         other => 'Other',
1181         wxp => 'Windows XP',
1182         w2k => 'Windows 2000',
1183         w2k3 =>, 'Windows 2003',
1184         w2k8 => 'Windows 2008',
1185         wvista => 'Windows Vista',
1186         win7 => 'Windows 7',
1187         win8 => 'Windows 8/2012',
1188         win10 => 'Windows 10/2016',
1189         l24 => 'Linux 2.4',
1190         l26 => 'Linux 2.6',
1191     };
1192 }
1193
1194 my $cdrom_path;
1195
1196 sub get_cdrom_path {
1197
1198     return  $cdrom_path if $cdrom_path;
1199
1200     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1201     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1202     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1203 }
1204
1205 sub get_iso_path {
1206     my ($storecfg, $vmid, $cdrom) = @_;
1207
1208     if ($cdrom eq 'cdrom') {
1209         return get_cdrom_path();
1210     } elsif ($cdrom eq 'none') {
1211         return '';
1212     } elsif ($cdrom =~ m|^/|) {
1213         return $cdrom;
1214     } else {
1215         return PVE::Storage::path($storecfg, $cdrom);
1216     }
1217 }
1218
1219 # try to convert old style file names to volume IDs
1220 sub filename_to_volume_id {
1221     my ($vmid, $file, $media) = @_;
1222
1223     if (!($file eq 'none' || $file eq 'cdrom' ||
1224           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1225
1226         return undef if $file =~ m|/|;
1227
1228         if ($media && $media eq 'cdrom') {
1229             $file = "local:iso/$file";
1230         } else {
1231             $file = "local:$vmid/$file";
1232         }
1233     }
1234
1235     return $file;
1236 }
1237
1238 sub verify_media_type {
1239     my ($opt, $vtype, $media) = @_;
1240
1241     return if !$media;
1242
1243     my $etype;
1244     if ($media eq 'disk') {
1245         $etype = 'images';
1246     } elsif ($media eq 'cdrom') {
1247         $etype = 'iso';
1248     } else {
1249         die "internal error";
1250     }
1251
1252     return if ($vtype eq $etype);
1253
1254     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1255 }
1256
1257 sub cleanup_drive_path {
1258     my ($opt, $storecfg, $drive) = @_;
1259
1260     # try to convert filesystem paths to volume IDs
1261
1262     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1263         ($drive->{file} !~ m|^/dev/.+|) &&
1264         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1265         ($drive->{file} !~ m/^\d+$/)) {
1266         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1267         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1268         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1269         verify_media_type($opt, $vtype, $drive->{media});
1270         $drive->{file} = $volid;
1271     }
1272
1273     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1274 }
1275
1276 sub parse_hotplug_features {
1277     my ($data) = @_;
1278
1279     my $res = {};
1280
1281     return $res if $data eq '0';
1282
1283     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1284
1285     foreach my $feature (PVE::Tools::split_list($data)) {
1286         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1287             $res->{$1} = 1;
1288         } else {
1289             die "invalid hotplug feature '$feature'\n";
1290         }
1291     }
1292     return $res;
1293 }
1294
1295 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1296 sub pve_verify_hotplug_features {
1297     my ($value, $noerr) = @_;
1298
1299     return $value if parse_hotplug_features($value);
1300
1301     return undef if $noerr;
1302
1303     die "unable to parse hotplug option\n";
1304 }
1305
1306 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1307 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1308 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1309 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1310 #        [,iothread=on][,serial=serial][,model=model]
1311
1312 sub parse_drive {
1313     my ($key, $data) = @_;
1314
1315     my ($interface, $index);
1316
1317     if ($key =~ m/^([^\d]+)(\d+)$/) {
1318         $interface = $1;
1319         $index = $2;
1320     } else {
1321         return undef;
1322     }
1323
1324     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1325                                      : $confdesc->{$key}->{format};
1326     if (!$desc) {
1327         warn "invalid drive key: $key\n";
1328         return undef;
1329     }
1330     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1331     return undef if !$res;
1332     $res->{interface} = $interface;
1333     $res->{index} = $index;
1334
1335     my $error = 0;
1336     foreach my $opt (qw(bps bps_rd bps_wr)) {
1337         if (my $bps = defined(delete $res->{$opt})) {
1338             if (defined($res->{"m$opt"})) {
1339                 warn "both $opt and m$opt specified\n";
1340                 ++$error;
1341                 next;
1342             }
1343             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1344         }
1345     }
1346
1347     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1348     for my $requirement (
1349         [bps_max_length => 'mbps_max'],
1350         [bps_rd_max_length => 'mbps_rd_max'],
1351         [bps_wr_max_length => 'mbps_wr_max'],
1352         [iops_max_length => 'iops_max'],
1353         [iops_rd_max_length => 'iops_rd_max'],
1354         [iops_wr_max_length => 'iops_wr_max']) {
1355         my ($option, $requires) = @$requirement;
1356         if ($res->{$option} && !$res->{$requires}) {
1357             warn "$option requires $requires\n";
1358             ++$error;
1359         }
1360     }
1361
1362     return undef if $error;
1363
1364     return undef if $res->{mbps_rd} && $res->{mbps};
1365     return undef if $res->{mbps_wr} && $res->{mbps};
1366     return undef if $res->{iops_rd} && $res->{iops};
1367     return undef if $res->{iops_wr} && $res->{iops};
1368
1369     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1370         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1371         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1372         return undef if $res->{interface} eq 'virtio';
1373     }
1374
1375     if (my $size = $res->{size}) {
1376         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1377     }
1378
1379     return $res;
1380 }
1381
1382 sub print_drive {
1383     my ($vmid, $drive) = @_;
1384     my $data = { %$drive };
1385     delete $data->{$_} for qw(index interface);
1386     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1387 }
1388
1389 sub scsi_inquiry {
1390     my($fh, $noerr) = @_;
1391
1392     my $SG_IO = 0x2285;
1393     my $SG_GET_VERSION_NUM = 0x2282;
1394
1395     my $versionbuf = "\x00" x 8;
1396     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1397     if (!$ret) {
1398         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1399         return undef;
1400     }
1401     my $version = unpack("I", $versionbuf);
1402     if ($version < 30000) {
1403         die "scsi generic interface too old\n"  if !$noerr;
1404         return undef;
1405     }
1406
1407     my $buf = "\x00" x 36;
1408     my $sensebuf = "\x00" x 8;
1409     my $cmd = pack("C x3 C x1", 0x12, 36);
1410
1411     # see /usr/include/scsi/sg.h
1412     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1413
1414     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1415                       length($sensebuf), 0, length($buf), $buf,
1416                       $cmd, $sensebuf, 6000);
1417
1418     $ret = ioctl($fh, $SG_IO, $packet);
1419     if (!$ret) {
1420         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1421         return undef;
1422     }
1423
1424     my @res = unpack($sg_io_hdr_t, $packet);
1425     if ($res[17] || $res[18]) {
1426         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my $res = {};
1431     (my $byte0, my $byte1, $res->{vendor},
1432      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1433
1434     $res->{removable} = $byte1 & 128 ? 1 : 0;
1435     $res->{type} = $byte0 & 31;
1436
1437     return $res;
1438 }
1439
1440 sub path_is_scsi {
1441     my ($path) = @_;
1442
1443     my $fh = IO::File->new("+<$path") || return undef;
1444     my $res = scsi_inquiry($fh, 1);
1445     close($fh);
1446
1447     return $res;
1448 }
1449
1450 sub machine_type_is_q35 {
1451     my ($conf) = @_;
1452
1453     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1454 }
1455
1456 sub print_tabletdevice_full {
1457     my ($conf) = @_;
1458
1459     my $q35 = machine_type_is_q35($conf);
1460
1461     # we use uhci for old VMs because tablet driver was buggy in older qemu
1462     my $usbbus = $q35 ? "ehci" : "uhci";
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_drivedevice_full {
1468     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1469
1470     my $device = '';
1471     my $maxdev = 0;
1472
1473     if ($drive->{interface} eq 'virtio') {
1474         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1475         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1476         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1477     } elsif ($drive->{interface} eq 'scsi') {
1478
1479         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1480         my $unit = $drive->{index} % $maxdev;
1481         my $devicetype = 'hd';
1482         my $path = '';
1483         if (drive_is_cdrom($drive)) {
1484             $devicetype = 'cd';
1485         } else {
1486             if ($drive->{file} =~ m|^/|) {
1487                 $path = $drive->{file};
1488                 if (my $info = path_is_scsi($path)) {
1489                     if ($info->{type} == 0) {
1490                         $devicetype = 'block';
1491                     } elsif ($info->{type} == 1) { # tape
1492                         $devicetype = 'generic';
1493                     }
1494                 }
1495             } else {
1496                  $path = PVE::Storage::path($storecfg, $drive->{file});
1497             }
1498
1499             if($path =~ m/^iscsi\:\/\//){
1500                 $devicetype = 'generic';
1501             }
1502         }
1503
1504         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1505             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1506         } else {
1507             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1508         }
1509
1510     } elsif ($drive->{interface} eq 'ide'){
1511         $maxdev = 2;
1512         my $controller = int($drive->{index} / $maxdev);
1513         my $unit = $drive->{index} % $maxdev;
1514         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1515
1516         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1517         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1518             $model = URI::Escape::uri_unescape($model);
1519             $device .= ",model=$model";
1520         }
1521     } elsif ($drive->{interface} eq 'sata'){
1522         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1523         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1524         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1525     } elsif ($drive->{interface} eq 'usb') {
1526         die "implement me";
1527         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1528     } else {
1529         die "unsupported interface type";
1530     }
1531
1532     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1533
1534     return $device;
1535 }
1536
1537 sub get_initiator_name {
1538     my $initiator;
1539
1540     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1541     while (defined(my $line = <$fh>)) {
1542         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1543         $initiator = $1;
1544         last;
1545     }
1546     $fh->close();
1547
1548     return $initiator;
1549 }
1550
1551 sub print_drive_full {
1552     my ($storecfg, $vmid, $drive) = @_;
1553
1554     my $path;
1555     my $volid = $drive->{file};
1556     my $format;
1557
1558     if (drive_is_cdrom($drive)) {
1559         $path = get_iso_path($storecfg, $vmid, $volid);
1560     } else {
1561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562         if ($storeid) {
1563             $path = PVE::Storage::path($storecfg, $volid);
1564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1565             $format = qemu_img_format($scfg, $volname);
1566         } else {
1567             $path = $volid;
1568             $format = "raw";
1569         }
1570    }
1571
1572     my $opts = '';
1573     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1574     foreach my $o (@qemu_drive_options) {
1575         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1576     }
1577     if (my $serial = $drive->{serial}) {
1578         $serial = URI::Escape::uri_unescape($serial);
1579         $opts .= ",serial=$serial";
1580     }
1581
1582     $opts .= ",format=$format" if $format && !$drive->{format};
1583
1584     foreach my $o (qw(bps bps_rd bps_wr)) {
1585         my $v = $drive->{"m$o"};
1586         $opts .= ",$o=" . int($v*1024*1024) if $v;
1587     }
1588
1589     my $cache_direct = 0;
1590
1591     if (my $cache = $drive->{cache}) {
1592         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1593     } elsif (!drive_is_cdrom($drive)) {
1594         $opts .= ",cache=none";
1595         $cache_direct = 1;
1596     }
1597
1598     # aio native works only with O_DIRECT
1599     if (!$drive->{aio}) {
1600         if($cache_direct) {
1601             $opts .= ",aio=native";
1602         } else {
1603             $opts .= ",aio=threads";
1604         }
1605     }
1606
1607     if (!drive_is_cdrom($drive)) {
1608         my $detectzeroes;
1609         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1610             $detectzeroes = 'off';
1611         } elsif ($drive->{discard}) {
1612             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1613         } else {
1614             # This used to be our default with discard not being specified:
1615             $detectzeroes = 'on';
1616         }
1617         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1618     }
1619
1620     my $pathinfo = $path ? "file=$path," : '';
1621
1622     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1623 }
1624
1625 sub print_netdevice_full {
1626     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1627
1628     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1629
1630     my $device = $net->{model};
1631     if ($net->{model} eq 'virtio') {
1632          $device = 'virtio-net-pci';
1633      };
1634
1635     my $pciaddr = print_pci_addr("$netid", $bridges);
1636     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1637     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1638         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1639         my $vectors = $net->{queues} * 2 + 2;
1640         $tmpstr .= ",vectors=$vectors,mq=on";
1641     }
1642     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1643
1644     if ($use_old_bios_files) {
1645         my $romfile;
1646         if ($device eq 'virtio-net-pci') {
1647             $romfile = 'pxe-virtio.rom';
1648         } elsif ($device eq 'e1000') {
1649             $romfile = 'pxe-e1000.rom';
1650         } elsif ($device eq 'ne2k') {
1651             $romfile = 'pxe-ne2k_pci.rom';
1652         } elsif ($device eq 'pcnet') {
1653             $romfile = 'pxe-pcnet.rom';
1654         } elsif ($device eq 'rtl8139') {
1655             $romfile = 'pxe-rtl8139.rom';
1656         }
1657         $tmpstr .= ",romfile=$romfile" if $romfile;
1658     }
1659
1660     return $tmpstr;
1661 }
1662
1663 sub print_netdev_full {
1664     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1665
1666     my $i = '';
1667     if ($netid =~ m/^net(\d+)$/) {
1668         $i = int($1);
1669     }
1670
1671     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1672
1673     my $ifname = "tap${vmid}i$i";
1674
1675     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1676     die "interface name '$ifname' is too long (max 15 character)\n"
1677         if length($ifname) >= 16;
1678
1679     my $vhostparam = '';
1680     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1681
1682     my $vmname = $conf->{name} || "vm$vmid";
1683
1684     my $netdev = "";
1685     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1686
1687     if ($net->{bridge}) {
1688         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1689     } else {
1690         $netdev = "type=user,id=$netid,hostname=$vmname";
1691     }
1692
1693     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1694
1695     return $netdev;
1696 }
1697
1698
1699 sub print_cpu_device {
1700     my ($conf, $id) = @_;
1701
1702     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1703     my $cpu = $nokvm ? "qemu64" : "kvm64";
1704     if (my $cputype = $conf->{cpu}) {
1705         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1706             or die "Cannot parse cpu description: $cputype\n";
1707         $cpu = $cpuconf->{cputype};
1708     }
1709
1710     my $sockets = 1;
1711     $sockets = $conf->{sockets} if  $conf->{sockets};
1712     my $cores = $conf->{cores} || 1;
1713
1714     my $current_core = ($id - 1) % $cores;
1715     my $current_socket = int(($id - $current_core)/$cores);
1716
1717     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1718 }
1719
1720 sub drive_is_cdrom {
1721     my ($drive) = @_;
1722
1723     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1724
1725 }
1726
1727 sub parse_number_sets {
1728     my ($set) = @_;
1729     my $res = [];
1730     foreach my $part (split(/;/, $set)) {
1731         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1732             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1733             push @$res, [ $1, $2 ];
1734         } else {
1735             die "invalid range: $part\n";
1736         }
1737     }
1738     return $res;
1739 }
1740
1741 sub parse_numa {
1742     my ($data) = @_;
1743
1744     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1745     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1746     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1747     return $res;
1748 }
1749
1750 sub parse_hostpci {
1751     my ($value) = @_;
1752
1753     return undef if !$value;
1754
1755     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1756
1757     my @idlist = split(/;/, $res->{host});
1758     delete $res->{host};
1759     foreach my $id (@idlist) {
1760         if ($id =~ /^$PCIRE$/) {
1761             if (defined($2)) {
1762                 push @{$res->{pciid}}, { id => $1, function => $2 };
1763             } else {
1764                 my $pcidevices = lspci($1);
1765                 $res->{pciid} = $pcidevices->{$1};
1766             }
1767         } else {
1768             # should have been caught by parse_property_string already
1769             die "failed to parse PCI id: $id\n";
1770         }
1771     }
1772     return $res;
1773 }
1774
1775 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1776 sub parse_net {
1777     my ($data) = @_;
1778
1779     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1780     if ($@) {
1781         warn $@;
1782         return undef;
1783     }
1784     if (!defined($res->{macaddr})) {
1785         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1786         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1787     }
1788     return $res;
1789 }
1790
1791 sub print_net {
1792     my $net = shift;
1793
1794     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1795 }
1796
1797 sub add_random_macs {
1798     my ($settings) = @_;
1799
1800     foreach my $opt (keys %$settings) {
1801         next if $opt !~ m/^net(\d+)$/;
1802         my $net = parse_net($settings->{$opt});
1803         next if !$net;
1804         $settings->{$opt} = print_net($net);
1805     }
1806 }
1807
1808 sub vm_is_volid_owner {
1809     my ($storecfg, $vmid, $volid) = @_;
1810
1811     if ($volid !~  m|^/|) {
1812         my ($path, $owner);
1813         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1814         if ($owner && ($owner == $vmid)) {
1815             return 1;
1816         }
1817     }
1818
1819     return undef;
1820 }
1821
1822 sub split_flagged_list {
1823     my $text = shift || '';
1824     $text =~ s/[,;]/ /g;
1825     $text =~ s/^\s+//;
1826     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1827 }
1828
1829 sub join_flagged_list {
1830     my ($how, $lst) = @_;
1831     join $how, map { $lst->{$_} . $_ } keys %$lst;
1832 }
1833
1834 sub vmconfig_delete_pending_option {
1835     my ($conf, $key, $force) = @_;
1836
1837     delete $conf->{pending}->{$key};
1838     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1839     $pending_delete_hash->{$key} = $force ? '!' : '';
1840     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1841 }
1842
1843 sub vmconfig_undelete_pending_option {
1844     my ($conf, $key) = @_;
1845
1846     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1847     delete $pending_delete_hash->{$key};
1848
1849     if (%$pending_delete_hash) {
1850         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1851     } else {
1852         delete $conf->{pending}->{delete};
1853     }
1854 }
1855
1856 sub vmconfig_register_unused_drive {
1857     my ($storecfg, $vmid, $conf, $drive) = @_;
1858
1859     if (!drive_is_cdrom($drive)) {
1860         my $volid = $drive->{file};
1861         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1862             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1863         }
1864     }
1865 }
1866
1867 sub vmconfig_cleanup_pending {
1868     my ($conf) = @_;
1869
1870     # remove pending changes when nothing changed
1871     my $changes;
1872     foreach my $opt (keys %{$conf->{pending}}) {
1873         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1874             $changes = 1;
1875             delete $conf->{pending}->{$opt};
1876         }
1877     }
1878
1879     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1880     my $pending_delete_hash = {};
1881     while (my ($opt, $force) = each %$current_delete_hash) {
1882         if (defined($conf->{$opt})) {
1883             $pending_delete_hash->{$opt} = $force;
1884         } else {
1885             $changes = 1;
1886         }
1887     }
1888
1889     if (%$pending_delete_hash) {
1890         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1891     } else {
1892         delete $conf->{pending}->{delete};
1893     }
1894
1895     return $changes;
1896 }
1897
1898 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1899 my $smbios1_fmt = {
1900     uuid => {
1901         type => 'string',
1902         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1903         format_description => 'UUID',
1904         description => "Set SMBIOS1 UUID.",
1905         optional => 1,
1906     },
1907     version => {
1908         type => 'string',
1909         pattern => '\S+',
1910         format_description => 'string',
1911         description => "Set SMBIOS1 version.",
1912         optional => 1,
1913     },
1914     serial => {
1915         type => 'string',
1916         pattern => '\S+',
1917         format_description => 'string',
1918         description => "Set SMBIOS1 serial number.",
1919         optional => 1,
1920     },
1921     manufacturer => {
1922         type => 'string',
1923         pattern => '\S+',
1924         format_description => 'string',
1925         description => "Set SMBIOS1 manufacturer.",
1926         optional => 1,
1927     },
1928     product => {
1929         type => 'string',
1930         pattern => '\S+',
1931         format_description => 'string',
1932         description => "Set SMBIOS1 product ID.",
1933         optional => 1,
1934     },
1935     sku => {
1936         type => 'string',
1937         pattern => '\S+',
1938         format_description => 'string',
1939         description => "Set SMBIOS1 SKU string.",
1940         optional => 1,
1941     },
1942     family => {
1943         type => 'string',
1944         pattern => '\S+',
1945         format_description => 'string',
1946         description => "Set SMBIOS1 family string.",
1947         optional => 1,
1948     },
1949 };
1950
1951 sub parse_smbios1 {
1952     my ($data) = @_;
1953
1954     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1955     warn $@ if $@;
1956     return $res;
1957 }
1958
1959 sub print_smbios1 {
1960     my ($smbios1) = @_;
1961     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1962 }
1963
1964 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1965
1966 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1967 sub verify_bootdisk {
1968     my ($value, $noerr) = @_;
1969
1970     return $value if is_valid_drivename($value);
1971
1972     return undef if $noerr;
1973
1974     die "invalid boot disk '$value'\n";
1975 }
1976
1977 sub parse_watchdog {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1983     warn $@ if $@;
1984     return $res;
1985 }
1986
1987 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1988 sub verify_usb_device {
1989     my ($value, $noerr) = @_;
1990
1991     return $value if parse_usb_device($value);
1992
1993     return undef if $noerr;
1994
1995     die "unable to parse usb device\n";
1996 }
1997
1998 # add JSON properties for create and set function
1999 sub json_config_properties {
2000     my $prop = shift;
2001
2002     foreach my $opt (keys %$confdesc) {
2003         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2004         $prop->{$opt} = $confdesc->{$opt};
2005     }
2006
2007     return $prop;
2008 }
2009
2010 sub check_type {
2011     my ($key, $value) = @_;
2012
2013     die "unknown setting '$key'\n" if !$confdesc->{$key};
2014
2015     my $type = $confdesc->{$key}->{type};
2016
2017     if (!defined($value)) {
2018         die "got undefined value\n";
2019     }
2020
2021     if ($value =~ m/[\n\r]/) {
2022         die "property contains a line feed\n";
2023     }
2024
2025     if ($type eq 'boolean') {
2026         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2027         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2028         die "type check ('boolean') failed - got '$value'\n";
2029     } elsif ($type eq 'integer') {
2030         return int($1) if $value =~ m/^(\d+)$/;
2031         die "type check ('integer') failed - got '$value'\n";
2032     } elsif ($type eq 'number') {
2033         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2034         die "type check ('number') failed - got '$value'\n";
2035     } elsif ($type eq 'string') {
2036         if (my $fmt = $confdesc->{$key}->{format}) {
2037             PVE::JSONSchema::check_format($fmt, $value);
2038             return $value;
2039         }
2040         $value =~ s/^\"(.*)\"$/$1/;
2041         return $value;
2042     } else {
2043         die "internal error"
2044     }
2045 }
2046
2047 sub check_iommu_support{
2048     #fixme : need to check IOMMU support
2049     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2050
2051     my $iommu=1;
2052     return $iommu;
2053
2054 }
2055
2056 sub touch_config {
2057     my ($vmid) = @_;
2058
2059     my $conf = PVE::QemuConfig->config_file($vmid);
2060     utime undef, undef, $conf;
2061 }
2062
2063 sub destroy_vm {
2064     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2065
2066     my $conffile = PVE::QemuConfig->config_file($vmid);
2067
2068     my $conf = PVE::QemuConfig->load_config($vmid);
2069
2070     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2071
2072     # only remove disks owned by this VM
2073     foreach_drive($conf, sub {
2074         my ($ds, $drive) = @_;
2075
2076         return if drive_is_cdrom($drive);
2077
2078         my $volid = $drive->{file};
2079
2080         return if !$volid || $volid =~ m|^/|;
2081
2082         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2083         return if !$path || !$owner || ($owner != $vmid);
2084
2085         eval {
2086             PVE::Storage::vdisk_free($storecfg, $volid);
2087         };
2088         warn "Could not remove disk '$volid', check manually: $@" if $@;
2089
2090     });
2091
2092     if ($keep_empty_config) {
2093         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2094     } else {
2095         unlink $conffile;
2096     }
2097
2098     # also remove unused disk
2099     eval {
2100         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2101
2102         eval {
2103             PVE::Storage::foreach_volid($dl, sub {
2104                 my ($volid, $sid, $volname, $d) = @_;
2105                 PVE::Storage::vdisk_free($storecfg, $volid);
2106             });
2107         };
2108         warn $@ if $@;
2109
2110     };
2111     warn $@ if $@;
2112 }
2113
2114 sub parse_vm_config {
2115     my ($filename, $raw) = @_;
2116
2117     return undef if !defined($raw);
2118
2119     my $res = {
2120         digest => Digest::SHA::sha1_hex($raw),
2121         snapshots => {},
2122         pending => {},
2123     };
2124
2125     $filename =~ m|/qemu-server/(\d+)\.conf$|
2126         || die "got strange filename '$filename'";
2127
2128     my $vmid = $1;
2129
2130     my $conf = $res;
2131     my $descr;
2132     my $section = '';
2133
2134     my @lines = split(/\n/, $raw);
2135     foreach my $line (@lines) {
2136         next if $line =~ m/^\s*$/;
2137
2138         if ($line =~ m/^\[PENDING\]\s*$/i) {
2139             $section = 'pending';
2140             if (defined($descr)) {
2141                 $descr =~ s/\s+$//;
2142                 $conf->{description} = $descr;
2143             }
2144             $descr = undef;
2145             $conf = $res->{$section} = {};
2146             next;
2147
2148         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2149             $section = $1;
2150             if (defined($descr)) {
2151                 $descr =~ s/\s+$//;
2152                 $conf->{description} = $descr;
2153             }
2154             $descr = undef;
2155             $conf = $res->{snapshots}->{$section} = {};
2156             next;
2157         }
2158
2159         if ($line =~ m/^\#(.*)\s*$/) {
2160             $descr = '' if !defined($descr);
2161             $descr .= PVE::Tools::decode_text($1) . "\n";
2162             next;
2163         }
2164
2165         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2166             $descr = '' if !defined($descr);
2167             $descr .= PVE::Tools::decode_text($2);
2168         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2169             $conf->{snapstate} = $1;
2170         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2171             my $key = $1;
2172             my $value = $2;
2173             $conf->{$key} = $value;
2174         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2175             my $value = $1;
2176             if ($section eq 'pending') {
2177                 $conf->{delete} = $value; # we parse this later
2178             } else {
2179                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2180             }
2181         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2182             my $key = $1;
2183             my $value = $2;
2184             eval { $value = check_type($key, $value); };
2185             if ($@) {
2186                 warn "vm $vmid - unable to parse value of '$key' - $@";
2187             } else {
2188                 $key = 'ide2' if $key eq 'cdrom';
2189                 my $fmt = $confdesc->{$key}->{format};
2190                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2191                     my $v = parse_drive($key, $value);
2192                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2193                         $v->{file} = $volid;
2194                         $value = print_drive($vmid, $v);
2195                     } else {
2196                         warn "vm $vmid - unable to parse value of '$key'\n";
2197                         next;
2198                     }
2199                 }
2200
2201                 $conf->{$key} = $value;
2202             }
2203         }
2204     }
2205
2206     if (defined($descr)) {
2207         $descr =~ s/\s+$//;
2208         $conf->{description} = $descr;
2209     }
2210     delete $res->{snapstate}; # just to be sure
2211
2212     return $res;
2213 }
2214
2215 sub write_vm_config {
2216     my ($filename, $conf) = @_;
2217
2218     delete $conf->{snapstate}; # just to be sure
2219
2220     if ($conf->{cdrom}) {
2221         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2222         $conf->{ide2} = $conf->{cdrom};
2223         delete $conf->{cdrom};
2224     }
2225
2226     # we do not use 'smp' any longer
2227     if ($conf->{sockets}) {
2228         delete $conf->{smp};
2229     } elsif ($conf->{smp}) {
2230         $conf->{sockets} = $conf->{smp};
2231         delete $conf->{cores};
2232         delete $conf->{smp};
2233     }
2234
2235     my $used_volids = {};
2236
2237     my $cleanup_config = sub {
2238         my ($cref, $pending, $snapname) = @_;
2239
2240         foreach my $key (keys %$cref) {
2241             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2242                 $key eq 'snapstate' || $key eq 'pending';
2243             my $value = $cref->{$key};
2244             if ($key eq 'delete') {
2245                 die "propertry 'delete' is only allowed in [PENDING]\n"
2246                     if !$pending;
2247                 # fixme: check syntax?
2248                 next;
2249             }
2250             eval { $value = check_type($key, $value); };
2251             die "unable to parse value of '$key' - $@" if $@;
2252
2253             $cref->{$key} = $value;
2254
2255             if (!$snapname && is_valid_drivename($key)) {
2256                 my $drive = parse_drive($key, $value);
2257                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2258             }
2259         }
2260     };
2261
2262     &$cleanup_config($conf);
2263
2264     &$cleanup_config($conf->{pending}, 1);
2265
2266     foreach my $snapname (keys %{$conf->{snapshots}}) {
2267         die "internal error" if $snapname eq 'pending';
2268         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2269     }
2270
2271     # remove 'unusedX' settings if we re-add a volume
2272     foreach my $key (keys %$conf) {
2273         my $value = $conf->{$key};
2274         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2275             delete $conf->{$key};
2276         }
2277     }
2278
2279     my $generate_raw_config = sub {
2280         my ($conf, $pending) = @_;
2281
2282         my $raw = '';
2283
2284         # add description as comment to top of file
2285         if (defined(my $descr = $conf->{description})) {
2286             if ($descr) {
2287                 foreach my $cl (split(/\n/, $descr)) {
2288                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2289                 }
2290             } else {
2291                 $raw .= "#\n" if $pending;
2292             }
2293         }
2294
2295         foreach my $key (sort keys %$conf) {
2296             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2297             $raw .= "$key: $conf->{$key}\n";
2298         }
2299         return $raw;
2300     };
2301
2302     my $raw = &$generate_raw_config($conf);
2303
2304     if (scalar(keys %{$conf->{pending}})){
2305         $raw .= "\n[PENDING]\n";
2306         $raw .= &$generate_raw_config($conf->{pending}, 1);
2307     }
2308
2309     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2310         $raw .= "\n[$snapname]\n";
2311         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2312     }
2313
2314     return $raw;
2315 }
2316
2317 sub load_defaults {
2318
2319     my $res = {};
2320
2321     # we use static defaults from our JSON schema configuration
2322     foreach my $key (keys %$confdesc) {
2323         if (defined(my $default = $confdesc->{$key}->{default})) {
2324             $res->{$key} = $default;
2325         }
2326     }
2327
2328     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2329     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2330
2331     return $res;
2332 }
2333
2334 sub config_list {
2335     my $vmlist = PVE::Cluster::get_vmlist();
2336     my $res = {};
2337     return $res if !$vmlist || !$vmlist->{ids};
2338     my $ids = $vmlist->{ids};
2339
2340     foreach my $vmid (keys %$ids) {
2341         my $d = $ids->{$vmid};
2342         next if !$d->{node} || $d->{node} ne $nodename;
2343         next if !$d->{type} || $d->{type} ne 'qemu';
2344         $res->{$vmid}->{exists} = 1;
2345     }
2346     return $res;
2347 }
2348
2349 # test if VM uses local resources (to prevent migration)
2350 sub check_local_resources {
2351     my ($conf, $noerr) = @_;
2352
2353     my $loc_res = 0;
2354
2355     $loc_res = 1 if $conf->{hostusb}; # old syntax
2356     $loc_res = 1 if $conf->{hostpci}; # old syntax
2357
2358     foreach my $k (keys %$conf) {
2359         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2360         # sockets are safe: they will recreated be on the target side post-migrate
2361         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2362         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2363     }
2364
2365     die "VM uses local resources\n" if $loc_res && !$noerr;
2366
2367     return $loc_res;
2368 }
2369
2370 # check if used storages are available on all nodes (use by migrate)
2371 sub check_storage_availability {
2372     my ($storecfg, $conf, $node) = @_;
2373
2374     foreach_drive($conf, sub {
2375         my ($ds, $drive) = @_;
2376
2377         my $volid = $drive->{file};
2378         return if !$volid;
2379
2380         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2381         return if !$sid;
2382
2383         # check if storage is available on both nodes
2384         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2385         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2386    });
2387 }
2388
2389 # list nodes where all VM images are available (used by has_feature API)
2390 sub shared_nodes {
2391     my ($conf, $storecfg) = @_;
2392
2393     my $nodelist = PVE::Cluster::get_nodelist();
2394     my $nodehash = { map { $_ => 1 } @$nodelist };
2395     my $nodename = PVE::INotify::nodename();
2396
2397     foreach_drive($conf, sub {
2398         my ($ds, $drive) = @_;
2399
2400         my $volid = $drive->{file};
2401         return if !$volid;
2402
2403         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2404         if ($storeid) {
2405             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2406             if ($scfg->{disable}) {
2407                 $nodehash = {};
2408             } elsif (my $avail = $scfg->{nodes}) {
2409                 foreach my $node (keys %$nodehash) {
2410                     delete $nodehash->{$node} if !$avail->{$node};
2411                 }
2412             } elsif (!$scfg->{shared}) {
2413                 foreach my $node (keys %$nodehash) {
2414                     delete $nodehash->{$node} if $node ne $nodename
2415                 }
2416             }
2417         }
2418     });
2419
2420     return $nodehash
2421 }
2422
2423 sub check_cmdline {
2424     my ($pidfile, $pid) = @_;
2425
2426     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2427     if (defined($fh)) {
2428         my $line = <$fh>;
2429         $fh->close;
2430         return undef if !$line;
2431         my @param = split(/\0/, $line);
2432
2433         my $cmd = $param[0];
2434         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2435
2436         for (my $i = 0; $i < scalar (@param); $i++) {
2437             my $p = $param[$i];
2438             next if !$p;
2439             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2440                 my $p = $param[$i+1];
2441                 return 1 if $p && ($p eq $pidfile);
2442                 return undef;
2443             }
2444         }
2445     }
2446     return undef;
2447 }
2448
2449 sub check_running {
2450     my ($vmid, $nocheck, $node) = @_;
2451
2452     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2453
2454     die "unable to find configuration file for VM $vmid - no such machine\n"
2455         if !$nocheck && ! -f $filename;
2456
2457     my $pidfile = pidfile_name($vmid);
2458
2459     if (my $fd = IO::File->new("<$pidfile")) {
2460         my $st = stat($fd);
2461         my $line = <$fd>;
2462         close($fd);
2463
2464         my $mtime = $st->mtime;
2465         if ($mtime > time()) {
2466             warn "file '$filename' modified in future\n";
2467         }
2468
2469         if ($line =~ m/^(\d+)$/) {
2470             my $pid = $1;
2471             if (check_cmdline($pidfile, $pid)) {
2472                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2473                     return $pid;
2474                 }
2475             }
2476         }
2477     }
2478
2479     return undef;
2480 }
2481
2482 sub vzlist {
2483
2484     my $vzlist = config_list();
2485
2486     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2487
2488     while (defined(my $de = $fd->read)) {
2489         next if $de !~ m/^(\d+)\.pid$/;
2490         my $vmid = $1;
2491         next if !defined($vzlist->{$vmid});
2492         if (my $pid = check_running($vmid)) {
2493             $vzlist->{$vmid}->{pid} = $pid;
2494         }
2495     }
2496
2497     return $vzlist;
2498 }
2499
2500 sub disksize {
2501     my ($storecfg, $conf) = @_;
2502
2503     my $bootdisk = $conf->{bootdisk};
2504     return undef if !$bootdisk;
2505     return undef if !is_valid_drivename($bootdisk);
2506
2507     return undef if !$conf->{$bootdisk};
2508
2509     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2510     return undef if !defined($drive);
2511
2512     return undef if drive_is_cdrom($drive);
2513
2514     my $volid = $drive->{file};
2515     return undef if !$volid;
2516
2517     return $drive->{size};
2518 }
2519
2520 my $last_proc_pid_stat;
2521
2522 # get VM status information
2523 # This must be fast and should not block ($full == false)
2524 # We only query KVM using QMP if $full == true (this can be slow)
2525 sub vmstatus {
2526     my ($opt_vmid, $full) = @_;
2527
2528     my $res = {};
2529
2530     my $storecfg = PVE::Storage::config();
2531
2532     my $list = vzlist();
2533     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2534
2535     my $cpucount = $cpuinfo->{cpus} || 1;
2536
2537     foreach my $vmid (keys %$list) {
2538         next if $opt_vmid && ($vmid ne $opt_vmid);
2539
2540         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2541         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2542
2543         my $d = {};
2544         $d->{pid} = $list->{$vmid}->{pid};
2545
2546         # fixme: better status?
2547         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2548
2549         my $size = disksize($storecfg, $conf);
2550         if (defined($size)) {
2551             $d->{disk} = 0; # no info available
2552             $d->{maxdisk} = $size;
2553         } else {
2554             $d->{disk} = 0;
2555             $d->{maxdisk} = 0;
2556         }
2557
2558         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2559         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2560         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2561
2562         $d->{name} = $conf->{name} || "VM $vmid";
2563         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2564
2565         if ($conf->{balloon}) {
2566             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2567             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2568         }
2569
2570         $d->{uptime} = 0;
2571         $d->{cpu} = 0;
2572         $d->{mem} = 0;
2573
2574         $d->{netout} = 0;
2575         $d->{netin} = 0;
2576
2577         $d->{diskread} = 0;
2578         $d->{diskwrite} = 0;
2579
2580         $d->{template} = PVE::QemuConfig->is_template($conf);
2581
2582         $res->{$vmid} = $d;
2583     }
2584
2585     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2586     foreach my $dev (keys %$netdev) {
2587         next if $dev !~ m/^tap([1-9]\d*)i/;
2588         my $vmid = $1;
2589         my $d = $res->{$vmid};
2590         next if !$d;
2591
2592         $d->{netout} += $netdev->{$dev}->{receive};
2593         $d->{netin} += $netdev->{$dev}->{transmit};
2594
2595         if ($full) {
2596             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2597             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2598         }
2599
2600     }
2601
2602     my $ctime = gettimeofday;
2603
2604     foreach my $vmid (keys %$list) {
2605
2606         my $d = $res->{$vmid};
2607         my $pid = $d->{pid};
2608         next if !$pid;
2609
2610         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2611         next if !$pstat; # not running
2612
2613         my $used = $pstat->{utime} + $pstat->{stime};
2614
2615         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2616
2617         if ($pstat->{vsize}) {
2618             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2619         }
2620
2621         my $old = $last_proc_pid_stat->{$pid};
2622         if (!$old) {
2623             $last_proc_pid_stat->{$pid} = {
2624                 time => $ctime,
2625                 used => $used,
2626                 cpu => 0,
2627             };
2628             next;
2629         }
2630
2631         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2632
2633         if ($dtime > 1000) {
2634             my $dutime = $used -  $old->{used};
2635
2636             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2637             $last_proc_pid_stat->{$pid} = {
2638                 time => $ctime,
2639                 used => $used,
2640                 cpu => $d->{cpu},
2641             };
2642         } else {
2643             $d->{cpu} = $old->{cpu};
2644         }
2645     }
2646
2647     return $res if !$full;
2648
2649     my $qmpclient = PVE::QMPClient->new();
2650
2651     my $ballooncb = sub {
2652         my ($vmid, $resp) = @_;
2653
2654         my $info = $resp->{'return'};
2655         return if !$info->{max_mem};
2656
2657         my $d = $res->{$vmid};
2658
2659         # use memory assigned to VM
2660         $d->{maxmem} = $info->{max_mem};
2661         $d->{balloon} = $info->{actual};
2662
2663         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2664             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2665             $d->{freemem} = $info->{free_mem};
2666         }
2667
2668         $d->{ballooninfo} = $info;
2669     };
2670
2671     my $blockstatscb = sub {
2672         my ($vmid, $resp) = @_;
2673         my $data = $resp->{'return'} || [];
2674         my $totalrdbytes = 0;
2675         my $totalwrbytes = 0;
2676
2677         for my $blockstat (@$data) {
2678             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2679             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2680
2681             $blockstat->{device} =~ s/drive-//;
2682             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2683         }
2684         $res->{$vmid}->{diskread} = $totalrdbytes;
2685         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2686     };
2687
2688     my $statuscb = sub {
2689         my ($vmid, $resp) = @_;
2690
2691         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2692         # this fails if ballon driver is not loaded, so this must be
2693         # the last commnand (following command are aborted if this fails).
2694         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2695
2696         my $status = 'unknown';
2697         if (!defined($status = $resp->{'return'}->{status})) {
2698             warn "unable to get VM status\n";
2699             return;
2700         }
2701
2702         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2703     };
2704
2705     foreach my $vmid (keys %$list) {
2706         next if $opt_vmid && ($vmid ne $opt_vmid);
2707         next if !$res->{$vmid}->{pid}; # not running
2708         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2709     }
2710
2711     $qmpclient->queue_execute(undef, 2);
2712
2713     foreach my $vmid (keys %$list) {
2714         next if $opt_vmid && ($vmid ne $opt_vmid);
2715         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2716     }
2717
2718     return $res;
2719 }
2720
2721 sub foreach_drive {
2722     my ($conf, $func, @param) = @_;
2723
2724     foreach my $ds (valid_drive_names()) {
2725         next if !defined($conf->{$ds});
2726
2727         my $drive = parse_drive($ds, $conf->{$ds});
2728         next if !$drive;
2729
2730         &$func($ds, $drive, @param);
2731     }
2732 }
2733
2734 sub foreach_volid {
2735     my ($conf, $func, @param) = @_;
2736
2737     my $volhash = {};
2738
2739     my $test_volid = sub {
2740         my ($volid, $is_cdrom) = @_;
2741
2742         return if !$volid;
2743
2744         $volhash->{$volid} = $is_cdrom || 0;
2745     };
2746
2747     foreach_drive($conf, sub {
2748         my ($ds, $drive) = @_;
2749         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2750     });
2751
2752     foreach my $snapname (keys %{$conf->{snapshots}}) {
2753         my $snap = $conf->{snapshots}->{$snapname};
2754         &$test_volid($snap->{vmstate}, 0);
2755         foreach_drive($snap, sub {
2756             my ($ds, $drive) = @_;
2757             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2758         });
2759     }
2760
2761     foreach my $volid (keys %$volhash) {
2762         &$func($volid, $volhash->{$volid}, @param);
2763     }
2764 }
2765
2766 sub vga_conf_has_spice {
2767     my ($vga) = @_;
2768
2769     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2770
2771     return $1 || 1;
2772 }
2773
2774 sub config_to_command {
2775     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2776
2777     my $cmd = [];
2778     my $globalFlags = [];
2779     my $machineFlags = [];
2780     my $rtcFlags = [];
2781     my $cpuFlags = [];
2782     my $devices = [];
2783     my $pciaddr = '';
2784     my $bridges = {};
2785     my $kvmver = kvm_user_version();
2786     my $vernum = 0; # unknown
2787     my $ostype = $conf->{ostype};
2788     my $winversion = windows_version($ostype);
2789
2790     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2791         $vernum = $1*1000000+$2*1000;
2792     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2793         $vernum = $1*1000000+$2*1000+$3;
2794     }
2795
2796     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2797
2798     my $have_ovz = -f '/proc/vz/vestat';
2799
2800     my $q35 = machine_type_is_q35($conf);
2801     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2802     my $machine_type = $forcemachine || $conf->{machine};
2803     my $use_old_bios_files = undef;
2804     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2805
2806     my $cpuunits = defined($conf->{cpuunits}) ?
2807             $conf->{cpuunits} : $defaults->{cpuunits};
2808
2809     push @$cmd, '/usr/bin/kvm';
2810
2811     push @$cmd, '-id', $vmid;
2812
2813     my $use_virtio = 0;
2814
2815     my $qmpsocket = qmp_socket($vmid);
2816     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2817     push @$cmd, '-mon', "chardev=qmp,mode=control";
2818
2819
2820     push @$cmd, '-pidfile' , pidfile_name($vmid);
2821
2822     push @$cmd, '-daemonize';
2823
2824     if ($conf->{smbios1}) {
2825         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2826     }
2827
2828     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2829         my $ovmfbase;
2830
2831         # prefer the OVMF_CODE variant
2832         if (-f $OVMF_CODE) {
2833             $ovmfbase = $OVMF_CODE;
2834         } elsif (-f $OVMF_IMG) {
2835             $ovmfbase = $OVMF_IMG;
2836         }
2837
2838         die "no uefi base img found\n" if !$ovmfbase;
2839         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2840
2841         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2842             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2843             my $format = $d->{format} // 'raw';
2844             my $path;
2845             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2846             if ($storeid) {
2847                 $path = PVE::Storage::path($storecfg, $d->{file});
2848                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2849                 $format = qemu_img_format($scfg, $volname);
2850             } else {
2851                 $path = $d->{file};
2852                 $format = "raw";
2853             }
2854             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2855         } elsif ($ovmfbase eq $OVMF_CODE) {
2856             warn "using uefi without permanent efivars disk\n";
2857             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2858             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2859             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2860         } else {
2861             # if the base img is not OVMF_CODE, we do not have to bother
2862             # to create/use a vars image, since it will not be used anyway
2863             # this can only happen if someone manually deletes the OVMF_CODE image
2864             # or has an old pve-qemu-kvm version installed.
2865             # both should not happen, but we ignore it here
2866         }
2867     }
2868
2869
2870     # add usb controllers
2871     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2872     push @$devices, @usbcontrollers if @usbcontrollers;
2873     my $vga = $conf->{vga};
2874
2875     my $qxlnum = vga_conf_has_spice($vga);
2876     $vga = 'qxl' if $qxlnum;
2877
2878     if (!$vga) {
2879         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2880     }
2881
2882     # enable absolute mouse coordinates (needed by vnc)
2883     my $tablet;
2884     if (defined($conf->{tablet})) {
2885         $tablet = $conf->{tablet};
2886     } else {
2887         $tablet = $defaults->{tablet};
2888         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2889         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2890     }
2891
2892     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2893
2894     my $kvm_off = 0;
2895     my $gpu_passthrough;
2896
2897     # host pci devices
2898     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2899         my $d = parse_hostpci($conf->{"hostpci$i"});
2900         next if !$d;
2901
2902         my $pcie = $d->{pcie};
2903         if($pcie){
2904             die "q35 machine model is not enabled" if !$q35;
2905             $pciaddr = print_pcie_addr("hostpci$i");
2906         }else{
2907             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2908         }
2909
2910         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2911         my $xvga = '';
2912         if ($d->{'x-vga'}) {
2913             $xvga = ',x-vga=on';
2914             $kvm_off = 1;
2915             $vga = 'none';
2916             $gpu_passthrough = 1;
2917
2918             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2919                 $xvga = "";
2920             }
2921         }
2922         my $pcidevices = $d->{pciid};
2923         my $multifunction = 1 if @$pcidevices > 1;
2924
2925         my $j=0;
2926         foreach my $pcidevice (@$pcidevices) {
2927
2928             my $id = "hostpci$i";
2929             $id .= ".$j" if $multifunction;
2930             my $addr = $pciaddr;
2931             $addr .= ".$j" if $multifunction;
2932             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2933
2934             if($j == 0){
2935                 $devicestr .= "$rombar$xvga";
2936                 $devicestr .= ",multifunction=on" if $multifunction;
2937             }
2938
2939             push @$devices, '-device', $devicestr;
2940             $j++;
2941         }
2942     }
2943
2944     # usb devices
2945     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2946     push @$devices, @usbdevices if @usbdevices;
2947     # serial devices
2948     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2949         if (my $path = $conf->{"serial$i"}) {
2950             if ($path eq 'socket') {
2951                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2952                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2953                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2954             } else {
2955                 die "no such serial device\n" if ! -c $path;
2956                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2957                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2958             }
2959         }
2960     }
2961
2962     # parallel devices
2963     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2964         if (my $path = $conf->{"parallel$i"}) {
2965             die "no such parallel device\n" if ! -c $path;
2966             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2967             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2968             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2969         }
2970     }
2971
2972     my $vmname = $conf->{name} || "vm$vmid";
2973
2974     push @$cmd, '-name', $vmname;
2975
2976     my $sockets = 1;
2977     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2978     $sockets = $conf->{sockets} if  $conf->{sockets};
2979
2980     my $cores = $conf->{cores} || 1;
2981
2982     my $maxcpus = $sockets * $cores;
2983
2984     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2985
2986     my $allowed_vcpus = $cpuinfo->{cpus};
2987
2988     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2989         if ($allowed_vcpus < $maxcpus);
2990
2991     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2992
2993         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2994         for (my $i = 2; $i <= $vcpus; $i++)  {
2995             my $cpustr = print_cpu_device($conf,$i);
2996             push @$cmd, '-device', $cpustr;
2997         }
2998
2999     } else {
3000
3001         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3002     }
3003     push @$cmd, '-nodefaults';
3004
3005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3006
3007     my $bootindex_hash = {};
3008     my $i = 1;
3009     foreach my $o (split(//, $bootorder)) {
3010         $bootindex_hash->{$o} = $i*100;
3011         $i++;
3012     }
3013
3014     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3015
3016     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3017
3018     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3019
3020     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3021
3022     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3023         my $socket = vnc_socket($vmid);
3024         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3025     } else {
3026         push @$cmd, '-nographic';
3027     }
3028
3029     # time drift fix
3030     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3031
3032     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3033     my $useLocaltime = $conf->{localtime};
3034
3035     if ($winversion >= 5) { # windows
3036         $useLocaltime = 1 if !defined($conf->{localtime});
3037
3038         # use time drift fix when acpi is enabled
3039         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3040             $tdf = 1 if !defined($conf->{tdf});
3041         }
3042     }
3043
3044     if ($winversion >= 6) {
3045         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3046         push @$cmd, '-no-hpet';
3047     }
3048
3049     push @$rtcFlags, 'driftfix=slew' if $tdf;
3050
3051     if ($nokvm) {
3052         push @$machineFlags, 'accel=tcg';
3053     } else {
3054         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3055     }
3056
3057     if ($machine_type) {
3058         push @$machineFlags, "type=${machine_type}";
3059     }
3060
3061     if ($conf->{startdate}) {
3062         push @$rtcFlags, "base=$conf->{startdate}";
3063     } elsif ($useLocaltime) {
3064         push @$rtcFlags, 'base=localtime';
3065     }
3066
3067     my $cpu = $nokvm ? "qemu64" : "kvm64";
3068     if (my $cputype = $conf->{cpu}) {
3069         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3070             or die "Cannot parse cpu description: $cputype\n";
3071         $cpu = $cpuconf->{cputype};
3072         $kvm_off = 1 if $cpuconf->{hidden};
3073     }
3074
3075     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3076
3077     push @$cpuFlags , '-x2apic'
3078         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3079
3080     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3081
3082     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3083
3084     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3085
3086         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3087         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3088     }
3089
3090     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3091
3092     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3093
3094     push @$cpuFlags, 'kvm=off' if $kvm_off;
3095
3096     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3097         die "internal error"; # should not happen
3098
3099     push @$cpuFlags, "vendor=${cpu_vendor}"
3100         if $cpu_vendor ne 'default';
3101
3102     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3103
3104     push @$cmd, '-cpu', $cpu;
3105
3106     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3107
3108     push @$cmd, '-S' if $conf->{freeze};
3109
3110     # set keyboard layout
3111     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3112     push @$cmd, '-k', $kb if $kb;
3113
3114     # enable sound
3115     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3116     #push @$cmd, '-soundhw', 'es1370';
3117     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3118
3119     if($conf->{agent}) {
3120         my $qgasocket = qmp_socket($vmid, 1);
3121         my $pciaddr = print_pci_addr("qga0", $bridges);
3122         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3123         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3124         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3125     }
3126
3127     my $spice_port;
3128
3129     if ($qxlnum) {
3130         if ($qxlnum > 1) {
3131             if ($winversion){
3132                 for(my $i = 1; $i < $qxlnum; $i++){
3133                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3134                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3135                 }
3136             } else {
3137                 # assume other OS works like Linux
3138                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3139                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3140             }
3141         }
3142
3143         my $pciaddr = print_pci_addr("spice", $bridges);
3144
3145         my $nodename = PVE::INotify::nodename();
3146         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3147         $spice_port = PVE::Tools::next_spice_port($pfamily);
3148
3149         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3150
3151         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3152         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3153         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3154     }
3155
3156     # enable balloon by default, unless explicitly disabled
3157     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3158         $pciaddr = print_pci_addr("balloon0", $bridges);
3159         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3160     }
3161
3162     if ($conf->{watchdog}) {
3163         my $wdopts = parse_watchdog($conf->{watchdog});
3164         $pciaddr = print_pci_addr("watchdog", $bridges);
3165         my $watchdog = $wdopts->{model} || 'i6300esb';
3166         push @$devices, '-device', "$watchdog$pciaddr";
3167         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3168     }
3169
3170     my $vollist = [];
3171     my $scsicontroller = {};
3172     my $ahcicontroller = {};
3173     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3174
3175     # Add iscsi initiator name if available
3176     if (my $initiator = get_initiator_name()) {
3177         push @$devices, '-iscsi', "initiator-name=$initiator";
3178     }
3179
3180     foreach_drive($conf, sub {
3181         my ($ds, $drive) = @_;
3182
3183         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3184             push @$vollist, $drive->{file};
3185         }
3186
3187         $use_virtio = 1 if $ds =~ m/^virtio/;
3188
3189         if (drive_is_cdrom ($drive)) {
3190             if ($bootindex_hash->{d}) {
3191                 $drive->{bootindex} = $bootindex_hash->{d};
3192                 $bootindex_hash->{d} += 1;
3193             }
3194         } else {
3195             if ($bootindex_hash->{c}) {
3196                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3197                 $bootindex_hash->{c} += 1;
3198             }
3199         }
3200
3201         if($drive->{interface} eq 'virtio'){
3202            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3203         }
3204
3205         if ($drive->{interface} eq 'scsi') {
3206
3207             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3208
3209             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3210             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3211
3212             my $iothread = '';
3213             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3214                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3215                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3216             } elsif ($drive->{iothread}) {
3217                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3218             }
3219
3220             my $queues = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3222                 $queues = ",num_queues=$drive->{queues}";
3223             }
3224
3225             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3226             $scsicontroller->{$controller}=1;
3227         }
3228
3229         if ($drive->{interface} eq 'sata') {
3230            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3231            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3232            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3233            $ahcicontroller->{$controller}=1;
3234         }
3235
3236         if ($drive->{interface} eq 'efidisk') {
3237             # this will be added somewhere else
3238             return;
3239         }
3240
3241         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3242         push @$devices, '-drive',$drive_cmd;
3243         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3244     });
3245
3246     for (my $i = 0; $i < $MAX_NETS; $i++) {
3247          next if !$conf->{"net$i"};
3248          my $d = parse_net($conf->{"net$i"});
3249          next if !$d;
3250
3251          $use_virtio = 1 if $d->{model} eq 'virtio';
3252
3253          if ($bootindex_hash->{n}) {
3254             $d->{bootindex} = $bootindex_hash->{n};
3255             $bootindex_hash->{n} += 1;
3256          }
3257
3258          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3259          push @$devices, '-netdev', $netdevfull;
3260
3261          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3262          push @$devices, '-device', $netdevicefull;
3263     }
3264
3265     if (!$q35) {
3266         # add pci bridges
3267         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3268            $bridges->{1} = 1;
3269            $bridges->{2} = 1;
3270         }
3271
3272         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3273
3274         while (my ($k, $v) = each %$bridges) {
3275             $pciaddr = print_pci_addr("pci.$k");
3276             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3277         }
3278     }
3279
3280     # add custom args
3281     if ($conf->{args}) {
3282         my $aa = PVE::Tools::split_args($conf->{args});
3283         push @$cmd, @$aa;
3284     }
3285
3286     push @$cmd, @$devices;
3287     push @$cmd, '-rtc', join(',', @$rtcFlags)
3288         if scalar(@$rtcFlags);
3289     push @$cmd, '-machine', join(',', @$machineFlags)
3290         if scalar(@$machineFlags);
3291     push @$cmd, '-global', join(',', @$globalFlags)
3292         if scalar(@$globalFlags);
3293
3294     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3295 }
3296
3297 sub vnc_socket {
3298     my ($vmid) = @_;
3299     return "${var_run_tmpdir}/$vmid.vnc";
3300 }
3301
3302 sub spice_port {
3303     my ($vmid) = @_;
3304
3305     my $res = vm_mon_cmd($vmid, 'query-spice');
3306
3307     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3308 }
3309
3310 sub qmp_socket {
3311     my ($vmid, $qga) = @_;
3312     my $sockettype = $qga ? 'qga' : 'qmp';
3313     return "${var_run_tmpdir}/$vmid.$sockettype";
3314 }
3315
3316 sub pidfile_name {
3317     my ($vmid) = @_;
3318     return "${var_run_tmpdir}/$vmid.pid";
3319 }
3320
3321 sub vm_devices_list {
3322     my ($vmid) = @_;
3323
3324     my $res = vm_mon_cmd($vmid, 'query-pci');
3325     my $devices = {};
3326     foreach my $pcibus (@$res) {
3327         foreach my $device (@{$pcibus->{devices}}) {
3328             next if !$device->{'qdev_id'};
3329             if ($device->{'pci_bridge'}) {
3330                 $devices->{$device->{'qdev_id'}} = 1;
3331                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3332                     next if !$bridge_device->{'qdev_id'};
3333                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3334                     $devices->{$device->{'qdev_id'}}++;
3335                 }
3336             } else {
3337                 $devices->{$device->{'qdev_id'}} = 1;
3338             }
3339         }
3340     }
3341
3342     my $resblock = vm_mon_cmd($vmid, 'query-block');
3343     foreach my $block (@$resblock) {
3344         if($block->{device} =~ m/^drive-(\S+)/){
3345                 $devices->{$1} = 1;
3346         }
3347     }
3348
3349     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3350     foreach my $mice (@$resmice) {
3351         if ($mice->{name} eq 'QEMU HID Tablet') {
3352             $devices->{tablet} = 1;
3353             last;
3354         }
3355     }
3356
3357     # for usb devices there is no query-usb
3358     # but we can iterate over the entries in
3359     # qom-list path=/machine/peripheral
3360     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3361     foreach my $per (@$resperipheral) {
3362         if ($per->{name} =~ m/^usb\d+$/) {
3363             $devices->{$per->{name}} = 1;
3364         }
3365     }
3366
3367     return $devices;
3368 }
3369
3370 sub vm_deviceplug {
3371     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3372
3373     my $q35 = machine_type_is_q35($conf);
3374
3375     my $devices_list = vm_devices_list($vmid);
3376     return 1 if defined($devices_list->{$deviceid});
3377
3378     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3379
3380     if ($deviceid eq 'tablet') {
3381
3382         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3383
3384     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3385
3386         die "usb hotplug currently not reliable\n";
3387         # since we can't reliably hot unplug all added usb devices
3388         # and usb passthrough disables live migration
3389         # we disable usb hotplugging for now
3390         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3391
3392     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3393
3394         qemu_iothread_add($vmid, $deviceid, $device);
3395
3396         qemu_driveadd($storecfg, $vmid, $device);
3397         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3398
3399         qemu_deviceadd($vmid, $devicefull);
3400         eval { qemu_deviceaddverify($vmid, $deviceid); };
3401         if (my $err = $@) {
3402             eval { qemu_drivedel($vmid, $deviceid); };
3403             warn $@ if $@;
3404             die $err;
3405         }
3406
3407     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3408
3409
3410         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3411         my $pciaddr = print_pci_addr($deviceid);
3412         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3413
3414         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3417             qemu_iothread_add($vmid, $deviceid, $device);
3418             $devicefull .= ",iothread=iothread-$deviceid";
3419         }
3420
3421         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3422             $devicefull .= ",num_queues=$device->{queues}";
3423         }
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         qemu_deviceaddverify($vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3429
3430         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3431         qemu_driveadd($storecfg, $vmid, $device);
3432
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434         eval { qemu_deviceadd($vmid, $devicefull); };
3435         if (my $err = $@) {
3436             eval { qemu_drivedel($vmid, $deviceid); };
3437             warn $@ if $@;
3438             die $err;
3439         }
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3444
3445         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3446         my $use_old_bios_files = undef;
3447         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3450         qemu_deviceadd($vmid, $netdevicefull);
3451         eval { qemu_deviceaddverify($vmid, $deviceid); };
3452         if (my $err = $@) {
3453             eval { qemu_netdevdel($vmid, $deviceid); };
3454             warn $@ if $@;
3455             die $err;
3456         }
3457
3458     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3459
3460         my $bridgeid = $2;
3461         my $pciaddr = print_pci_addr($deviceid);
3462         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3463
3464         qemu_deviceadd($vmid, $devicefull);
3465         qemu_deviceaddverify($vmid, $deviceid);
3466
3467     } else {
3468         die "can't hotplug device '$deviceid'\n";
3469     }
3470
3471     return 1;
3472 }
3473
3474 # fixme: this should raise exceptions on error!
3475 sub vm_deviceunplug {
3476     my ($vmid, $conf, $deviceid) = @_;
3477
3478     my $devices_list = vm_devices_list($vmid);
3479     return 1 if !defined($devices_list->{$deviceid});
3480
3481     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3482
3483     if ($deviceid eq 'tablet') {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^usb\d+$/) {
3488
3489         die "usb hotplug currently not reliable\n";
3490         # when unplugging usb devices this way,
3491         # there may be remaining usb controllers/hubs
3492         # so we disable it for now
3493         qemu_devicedel($vmid, $deviceid);
3494         qemu_devicedelverify($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3497
3498         qemu_devicedel($vmid, $deviceid);
3499         qemu_devicedelverify($vmid, $deviceid);
3500         qemu_drivedel($vmid, $deviceid);
3501         qemu_iothread_del($conf, $vmid, $deviceid);
3502
3503     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3504
3505         qemu_devicedel($vmid, $deviceid);
3506         qemu_devicedelverify($vmid, $deviceid);
3507         qemu_iothread_del($conf, $vmid, $deviceid);
3508
3509     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3510
3511         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3512         my $device = parse_drive($deviceid, $conf->{$deviceid});
3513         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3514
3515         qemu_devicedel($vmid, $deviceid);
3516         qemu_drivedel($vmid, $deviceid);
3517         qemu_deletescsihw($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_netdevdel($vmid, $deviceid);
3524
3525     } else {
3526         die "can't unplug device '$deviceid'\n";
3527     }
3528
3529     return 1;
3530 }
3531
3532 sub qemu_deviceadd {
3533     my ($vmid, $devicefull) = @_;
3534
3535     $devicefull = "driver=".$devicefull;
3536     my %options =  split(/[=,]/, $devicefull);
3537
3538     vm_mon_cmd($vmid, "device_add" , %options);
3539 }
3540
3541 sub qemu_devicedel {
3542     my ($vmid, $deviceid) = @_;
3543
3544     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3545 }
3546
3547 sub qemu_iothread_add {
3548     my($vmid, $deviceid, $device) = @_;
3549
3550     if ($device->{iothread}) {
3551         my $iothreads = vm_iothreads_list($vmid);
3552         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3553     }
3554 }
3555
3556 sub qemu_iothread_del {
3557     my($conf, $vmid, $deviceid) = @_;
3558
3559     my $device = parse_drive($deviceid, $conf->{$deviceid});
3560     if ($device->{iothread}) {
3561         my $iothreads = vm_iothreads_list($vmid);
3562         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3563     }
3564 }
3565
3566 sub qemu_objectadd {
3567     my($vmid, $objectid, $qomtype) = @_;
3568
3569     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_objectdel {
3575     my($vmid, $objectid) = @_;
3576
3577     vm_mon_cmd($vmid, "object-del", id => $objectid);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_driveadd {
3583     my ($storecfg, $vmid, $device) = @_;
3584
3585     my $drive = print_drive_full($storecfg, $vmid, $device);
3586     $drive =~ s/\\/\\\\/g;
3587     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3588
3589     # If the command succeeds qemu prints: "OK"
3590     return 1 if $ret =~ m/OK/s;
3591
3592     die "adding drive failed: $ret\n";
3593 }
3594
3595 sub qemu_drivedel {
3596     my($vmid, $deviceid) = @_;
3597
3598     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3599     $ret =~ s/^\s+//;
3600
3601     return 1 if $ret eq "";
3602
3603     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3604     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3605
3606     die "deleting drive $deviceid failed : $ret\n";
3607 }
3608
3609 sub qemu_deviceaddverify {
3610     my ($vmid, $deviceid) = @_;
3611
3612     for (my $i = 0; $i <= 5; $i++) {
3613          my $devices_list = vm_devices_list($vmid);
3614          return 1 if defined($devices_list->{$deviceid});
3615          sleep 1;
3616     }
3617
3618     die "error on hotplug device '$deviceid'\n";
3619 }
3620
3621
3622 sub qemu_devicedelverify {
3623     my ($vmid, $deviceid) = @_;
3624
3625     # need to verify that the device is correctly removed as device_del
3626     # is async and empty return is not reliable
3627
3628     for (my $i = 0; $i <= 5; $i++) {
3629          my $devices_list = vm_devices_list($vmid);
3630          return 1 if !defined($devices_list->{$deviceid});
3631          sleep 1;
3632     }
3633
3634     die "error on hot-unplugging device '$deviceid'\n";
3635 }
3636
3637 sub qemu_findorcreatescsihw {
3638     my ($storecfg, $conf, $vmid, $device) = @_;
3639
3640     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3641
3642     my $scsihwid="$controller_prefix$controller";
3643     my $devices_list = vm_devices_list($vmid);
3644
3645     if(!defined($devices_list->{$scsihwid})) {
3646         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3647     }
3648
3649     return 1;
3650 }
3651
3652 sub qemu_deletescsihw {
3653     my ($conf, $vmid, $opt) = @_;
3654
3655     my $device = parse_drive($opt, $conf->{$opt});
3656
3657     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3658         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3659         return 1;
3660     }
3661
3662     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3663
3664     my $devices_list = vm_devices_list($vmid);
3665     foreach my $opt (keys %{$devices_list}) {
3666         if (PVE::QemuServer::is_valid_drivename($opt)) {
3667             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3668             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3669                 return 1;
3670             }
3671         }
3672     }
3673
3674     my $scsihwid="scsihw$controller";
3675
3676     vm_deviceunplug($vmid, $conf, $scsihwid);
3677
3678     return 1;
3679 }
3680
3681 sub qemu_add_pci_bridge {
3682     my ($storecfg, $conf, $vmid, $device) = @_;
3683
3684     my $bridges = {};
3685
3686     my $bridgeid;
3687
3688     print_pci_addr($device, $bridges);
3689
3690     while (my ($k, $v) = each %$bridges) {
3691         $bridgeid = $k;
3692     }
3693     return 1 if !defined($bridgeid) || $bridgeid < 1;
3694
3695     my $bridge = "pci.$bridgeid";
3696     my $devices_list = vm_devices_list($vmid);
3697
3698     if (!defined($devices_list->{$bridge})) {
3699         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3700     }
3701
3702     return 1;
3703 }
3704
3705 sub qemu_set_link_status {
3706     my ($vmid, $device, $up) = @_;
3707
3708     vm_mon_cmd($vmid, "set_link", name => $device,
3709                up => $up ? JSON::true : JSON::false);
3710 }
3711
3712 sub qemu_netdevadd {
3713     my ($vmid, $conf, $device, $deviceid) = @_;
3714
3715     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3716     my %options =  split(/[=,]/, $netdev);
3717
3718     vm_mon_cmd($vmid, "netdev_add",  %options);
3719     return 1;
3720 }
3721
3722 sub qemu_netdevdel {
3723     my ($vmid, $deviceid) = @_;
3724
3725     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3726 }
3727
3728 sub qemu_usb_hotplug {
3729     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3730
3731     return if !$device;
3732
3733     # remove the old one first
3734     vm_deviceunplug($vmid, $conf, $deviceid);
3735
3736     # check if xhci controller is necessary and available
3737     if ($device->{usb3}) {
3738
3739         my $devicelist = vm_devices_list($vmid);
3740
3741         if (!$devicelist->{xhci}) {
3742             my $pciaddr = print_pci_addr("xhci");
3743             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3744         }
3745     }
3746     my $d = parse_usb_device($device->{host});
3747     $d->{usb3} = $device->{usb3};
3748
3749     # add the new one
3750     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3751 }
3752
3753 sub qemu_cpu_hotplug {
3754     my ($vmid, $conf, $vcpus) = @_;
3755
3756     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3760     $sockets = $conf->{sockets} if  $conf->{sockets};
3761     my $cores = $conf->{cores} || 1;
3762     my $maxcpus = $sockets * $cores;
3763
3764     $vcpus = $maxcpus if !$vcpus;
3765
3766     die "you can't add more vcpus than maxcpus\n"
3767         if $vcpus > $maxcpus;
3768
3769     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3770
3771     if ($vcpus < $currentvcpus) {
3772
3773         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3774
3775             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3776                 qemu_devicedel($vmid, "cpu$i");
3777                 my $retry = 0;
3778                 my $currentrunningvcpus = undef;
3779                 while (1) {
3780                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3782                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3783                     $retry++;
3784                     sleep 1;
3785                 }
3786                 #update conf after each succesfull cpu unplug
3787                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3788                 PVE::QemuConfig->write_config($vmid, $conf);
3789             }
3790         } else {
3791             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3792         }
3793
3794         return;
3795     }
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm does not match its configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3802
3803         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3804             my $cpustr = print_cpu_device($conf, $i);
3805             qemu_deviceadd($vmid, $cpustr);
3806
3807             my $retry = 0;
3808             my $currentrunningvcpus = undef;
3809             while (1) {
3810                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3811                 last if scalar(@{$currentrunningvcpus}) == $i;
3812                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3813                 sleep 1;
3814                 $retry++;
3815             }
3816             #update conf after each succesfull cpu hotplug
3817             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3818             PVE::QemuConfig->write_config($vmid, $conf);
3819         }
3820     } else {
3821
3822         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3823             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3824         }
3825     }
3826 }
3827
3828 sub qemu_block_set_io_throttle {
3829     my ($vmid, $deviceid,
3830         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3831         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3832         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3833         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3834
3835     return if !check_running($vmid) ;
3836
3837     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3838         bps => int($bps),
3839         bps_rd => int($bps_rd),
3840         bps_wr => int($bps_wr),
3841         iops => int($iops),
3842         iops_rd => int($iops_rd),
3843         iops_wr => int($iops_wr),
3844         bps_max => int($bps_max),
3845         bps_rd_max => int($bps_rd_max),
3846         bps_wr_max => int($bps_wr_max),
3847         iops_max => int($iops_max),
3848         iops_rd_max => int($iops_rd_max),
3849         iops_wr_max => int($iops_wr_max),
3850         bps_max_length => int($bps_max_length),
3851         bps_rd_max_length => int($bps_rd_max_length),
3852         bps_wr_max_length => int($bps_wr_max_length),
3853         iops_max_length => int($iops_max_length),
3854         iops_rd_max_length => int($iops_rd_max_length),
3855         iops_wr_max_length => int($iops_wr_max_length),
3856     );
3857
3858 }
3859
3860 # old code, only used to shutdown old VM after update
3861 sub __read_avail {
3862     my ($fh, $timeout) = @_;
3863
3864     my $sel = new IO::Select;
3865     $sel->add($fh);
3866
3867     my $res = '';
3868     my $buf;
3869
3870     my @ready;
3871     while (scalar (@ready = $sel->can_read($timeout))) {
3872         my $count;
3873         if ($count = $fh->sysread($buf, 8192)) {
3874             if ($buf =~ /^(.*)\(qemu\) $/s) {
3875                 $res .= $1;
3876                 last;
3877             } else {
3878                 $res .= $buf;
3879             }
3880         } else {
3881             if (!defined($count)) {
3882                 die "$!\n";
3883             }
3884             last;
3885         }
3886     }
3887
3888     die "monitor read timeout\n" if !scalar(@ready);
3889
3890     return $res;
3891 }
3892
3893 # old code, only used to shutdown old VM after update
3894 sub vm_monitor_command {
3895     my ($vmid, $cmdstr, $nocheck) = @_;
3896
3897     my $res;
3898
3899     eval {
3900         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3901
3902         my $sname = "${var_run_tmpdir}/$vmid.mon";
3903
3904         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3905             die "unable to connect to VM $vmid socket - $!\n";
3906
3907         my $timeout = 3;
3908
3909         # hack: migrate sometime blocks the monitor (when migrate_downtime
3910         # is set)
3911         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3912             $timeout = 60*60; # 1 hour
3913         }
3914
3915         # read banner;
3916         my $data = __read_avail($sock, $timeout);
3917
3918         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3919             die "got unexpected qemu monitor banner\n";
3920         }
3921
3922         my $sel = new IO::Select;
3923         $sel->add($sock);
3924
3925         if (!scalar(my @ready = $sel->can_write($timeout))) {
3926             die "monitor write error - timeout";
3927         }
3928
3929         my $fullcmd = "$cmdstr\r";
3930
3931         # syslog('info', "VM $vmid monitor command: $cmdstr");
3932
3933         my $b;
3934         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3935             die "monitor write error - $!";
3936         }
3937
3938         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3939
3940         $timeout = 20;
3941
3942         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3943             $timeout = 60*60; # 1 hour
3944         } elsif ($cmdstr =~ m/^(eject|change)/) {
3945             $timeout = 60; # note: cdrom mount command is slow
3946         }
3947         if ($res = __read_avail($sock, $timeout)) {
3948
3949             my @lines = split("\r?\n", $res);
3950
3951             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3952
3953             $res = join("\n", @lines);
3954             $res .= "\n";
3955         }
3956     };
3957
3958     my $err = $@;
3959
3960     if ($err) {
3961         syslog("err", "VM $vmid monitor command failed - $err");
3962         die $err;
3963     }
3964
3965     return $res;
3966 }
3967
3968 sub qemu_block_resize {
3969     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3970
3971     my $running = check_running($vmid);
3972
3973     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3974
3975     return if !$running;
3976
3977     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3978
3979 }
3980
3981 sub qemu_volume_snapshot {
3982     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3983
3984     my $running = check_running($vmid);
3985
3986     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3987         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3988     } else {
3989         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3990     }
3991 }
3992
3993 sub qemu_volume_snapshot_delete {
3994     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3995
3996     my $running = check_running($vmid);
3997
3998     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3999         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4000     } else {
4001         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4002     }
4003 }
4004
4005 sub set_migration_caps {
4006     my ($vmid) = @_;
4007
4008     my $cap_ref = [];
4009
4010     my $enabled_cap = {
4011         "auto-converge" => 1,
4012         "xbzrle" => 1,
4013         "x-rdma-pin-all" => 0,
4014         "zero-blocks" => 0,
4015         "compress" => 0
4016     };
4017
4018     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4019
4020     for my $supported_capability (@$supported_capabilities) {
4021         push @$cap_ref, {
4022             capability => $supported_capability->{capability},
4023             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4024         };
4025     }
4026
4027     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4028 }
4029
4030 my $fast_plug_option = {
4031     'lock' => 1,
4032     'name' => 1,
4033     'onboot' => 1,
4034     'shares' => 1,
4035     'startup' => 1,
4036     'description' => 1,
4037     'protection' => 1,
4038 };
4039
4040 # hotplug changes in [PENDING]
4041 # $selection hash can be used to only apply specified options, for
4042 # example: { cores => 1 } (only apply changed 'cores')
4043 # $errors ref is used to return error messages
4044 sub vmconfig_hotplug_pending {
4045     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4046
4047     my $defaults = load_defaults();
4048
4049     # commit values which do not have any impact on running VM first
4050     # Note: those option cannot raise errors, we we do not care about
4051     # $selection and always apply them.
4052
4053     my $add_error = sub {
4054         my ($opt, $msg) = @_;
4055         $errors->{$opt} = "hotplug problem - $msg";
4056     };
4057
4058     my $changes = 0;
4059     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4060         if ($fast_plug_option->{$opt}) {
4061             $conf->{$opt} = $conf->{pending}->{$opt};
4062             delete $conf->{pending}->{$opt};
4063             $changes = 1;
4064         }
4065     }
4066
4067     if ($changes) {
4068         PVE::QemuConfig->write_config($vmid, $conf);
4069         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4070     }
4071
4072     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4073
4074     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4075     while (my ($opt, $force) = each %$pending_delete_hash) {
4076         next if $selection && !$selection->{$opt};
4077         eval {
4078             if ($opt eq 'hotplug') {
4079                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4080             } elsif ($opt eq 'tablet') {
4081                 die "skip\n" if !$hotplug_features->{usb};
4082                 if ($defaults->{tablet}) {
4083                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4084                 } else {
4085                     vm_deviceunplug($vmid, $conf, $opt);
4086                 }
4087             } elsif ($opt =~ m/^usb\d+/) {
4088                 die "skip\n";
4089                 # since we cannot reliably hot unplug usb devices
4090                 # we are disabling it
4091                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4092                 vm_deviceunplug($vmid, $conf, $opt);
4093             } elsif ($opt eq 'vcpus') {
4094                 die "skip\n" if !$hotplug_features->{cpu};
4095                 qemu_cpu_hotplug($vmid, $conf, undef);
4096             } elsif ($opt eq 'balloon') {
4097                 # enable balloon device is not hotpluggable
4098                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4099             } elsif ($fast_plug_option->{$opt}) {
4100                 # do nothing
4101             } elsif ($opt =~ m/^net(\d+)$/) {
4102                 die "skip\n" if !$hotplug_features->{network};
4103                 vm_deviceunplug($vmid, $conf, $opt);
4104             } elsif (is_valid_drivename($opt)) {
4105                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4106                 vm_deviceunplug($vmid, $conf, $opt);
4107                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4108             } elsif ($opt =~ m/^memory$/) {
4109                 die "skip\n" if !$hotplug_features->{memory};
4110                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4111             } elsif ($opt eq 'cpuunits') {
4112                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4113             } elsif ($opt eq 'cpulimit') {
4114                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4115             } else {
4116                 die "skip\n";
4117             }
4118         };
4119         if (my $err = $@) {
4120             &$add_error($opt, $err) if $err ne "skip\n";
4121         } else {
4122             # save new config if hotplug was successful
4123             delete $conf->{$opt};
4124             vmconfig_undelete_pending_option($conf, $opt);
4125             PVE::QemuConfig->write_config($vmid, $conf);
4126             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4127         }
4128     }
4129
4130     foreach my $opt (keys %{$conf->{pending}}) {
4131         next if $selection && !$selection->{$opt};
4132         my $value = $conf->{pending}->{$opt};
4133         eval {
4134             if ($opt eq 'hotplug') {
4135                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4136             } elsif ($opt eq 'tablet') {
4137                 die "skip\n" if !$hotplug_features->{usb};
4138                 if ($value == 1) {
4139                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4140                 } elsif ($value == 0) {
4141                     vm_deviceunplug($vmid, $conf, $opt);
4142                 }
4143             } elsif ($opt =~ m/^usb\d+$/) {
4144                 die "skip\n";
4145                 # since we cannot reliably hot unplug usb devices
4146                 # we are disabling it
4147                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4148                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4149                 die "skip\n" if !$d;
4150                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4151             } elsif ($opt eq 'vcpus') {
4152                 die "skip\n" if !$hotplug_features->{cpu};
4153                 qemu_cpu_hotplug($vmid, $conf, $value);
4154             } elsif ($opt eq 'balloon') {
4155                 # enable/disable balloning device is not hotpluggable
4156                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4157                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4158                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4159
4160                 # allow manual ballooning if shares is set to zero
4161                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4162                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4163                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4164                 }
4165             } elsif ($opt =~ m/^net(\d+)$/) {
4166                 # some changes can be done without hotplug
4167                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4168                                     $vmid, $opt, $value);
4169             } elsif (is_valid_drivename($opt)) {
4170                 # some changes can be done without hotplug
4171                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4172                                      $vmid, $opt, $value, 1);
4173             } elsif ($opt =~ m/^memory$/) { #dimms
4174                 die "skip\n" if !$hotplug_features->{memory};
4175                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4176             } elsif ($opt eq 'cpuunits') {
4177                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4178             } elsif ($opt eq 'cpulimit') {
4179                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4180                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4181             } else {
4182                 die "skip\n";  # skip non-hot-pluggable options
4183             }
4184         };
4185         if (my $err = $@) {
4186             &$add_error($opt, $err) if $err ne "skip\n";
4187         } else {
4188             # save new config if hotplug was successful
4189             $conf->{$opt} = $value;
4190             delete $conf->{pending}->{$opt};
4191             PVE::QemuConfig->write_config($vmid, $conf);
4192             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4193         }
4194     }
4195 }
4196
4197 sub try_deallocate_drive {
4198     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4199
4200     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4201         my $volid = $drive->{file};
4202         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4203             my $sid = PVE::Storage::parse_volume_id($volid);
4204             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4205
4206             # check if the disk is really unused
4207             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4208                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4209             PVE::Storage::vdisk_free($storecfg, $volid);
4210             return 1;
4211         } else {
4212             # If vm is not owner of this disk remove from config
4213             return 1;
4214         }
4215     }
4216
4217     return undef;
4218 }
4219
4220 sub vmconfig_delete_or_detach_drive {
4221     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4222
4223     my $drive = parse_drive($opt, $conf->{$opt});
4224
4225     my $rpcenv = PVE::RPCEnvironment::get();
4226     my $authuser = $rpcenv->get_user();
4227
4228     if ($force) {
4229         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4230         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4231     } else {
4232         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4233     }
4234 }
4235
4236 sub vmconfig_apply_pending {
4237     my ($vmid, $conf, $storecfg) = @_;
4238
4239     # cold plug
4240
4241     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4242     while (my ($opt, $force) = each %$pending_delete_hash) {
4243         die "internal error" if $opt =~ m/^unused/;
4244         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4245         if (!defined($conf->{$opt})) {
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             PVE::QemuConfig->write_config($vmid, $conf);
4248         } elsif (is_valid_drivename($opt)) {
4249             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4250             vmconfig_undelete_pending_option($conf, $opt);
4251             delete $conf->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253         } else {
4254             vmconfig_undelete_pending_option($conf, $opt);
4255             delete $conf->{$opt};
4256             PVE::QemuConfig->write_config($vmid, $conf);
4257         }
4258     }
4259
4260     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4261
4262     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4263         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4264
4265         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4266             # skip if nothing changed
4267         } elsif (is_valid_drivename($opt)) {
4268             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4269                 if defined($conf->{$opt});
4270             $conf->{$opt} = $conf->{pending}->{$opt};
4271         } else {
4272             $conf->{$opt} = $conf->{pending}->{$opt};
4273         }
4274
4275         delete $conf->{pending}->{$opt};
4276         PVE::QemuConfig->write_config($vmid, $conf);
4277     }
4278 }
4279
4280 my $safe_num_ne = sub {
4281     my ($a, $b) = @_;
4282
4283     return 0 if !defined($a) && !defined($b);
4284     return 1 if !defined($a);
4285     return 1 if !defined($b);
4286
4287     return $a != $b;
4288 };
4289
4290 my $safe_string_ne = sub {
4291     my ($a, $b) = @_;
4292
4293     return 0 if !defined($a) && !defined($b);
4294     return 1 if !defined($a);
4295     return 1 if !defined($b);
4296
4297     return $a ne $b;
4298 };
4299
4300 sub vmconfig_update_net {
4301     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4302
4303     my $newnet = parse_net($value);
4304
4305     if ($conf->{$opt}) {
4306         my $oldnet = parse_net($conf->{$opt});
4307
4308         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4309             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4310             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4311             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4312
4313             # for non online change, we try to hot-unplug
4314             die "skip\n" if !$hotplug;
4315             vm_deviceunplug($vmid, $conf, $opt);
4316         } else {
4317
4318             die "internal error" if $opt !~ m/net(\d+)/;
4319             my $iface = "tap${vmid}i$1";
4320
4321             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4322                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4323                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4324                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4325                 PVE::Network::tap_unplug($iface);
4326                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4327             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4328                 # Rate can be applied on its own but any change above needs to
4329                 # include the rate in tap_plug since OVS resets everything.
4330                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4331             }
4332
4333             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4334                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4335             }
4336
4337             return 1;
4338         }
4339     }
4340
4341     if ($hotplug) {
4342         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4343     } else {
4344         die "skip\n";
4345     }
4346 }
4347
4348 sub vmconfig_update_disk {
4349     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4350
4351     # fixme: do we need force?
4352
4353     my $drive = parse_drive($opt, $value);
4354
4355     if ($conf->{$opt}) {
4356
4357         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4358
4359             my $media = $drive->{media} || 'disk';
4360             my $oldmedia = $old_drive->{media} || 'disk';
4361             die "unable to change media type\n" if $media ne $oldmedia;
4362
4363             if (!drive_is_cdrom($old_drive)) {
4364
4365                 if ($drive->{file} ne $old_drive->{file}) {
4366
4367                     die "skip\n" if !$hotplug;
4368
4369                     # unplug and register as unused
4370                     vm_deviceunplug($vmid, $conf, $opt);
4371                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4372
4373                 } else {
4374                     # update existing disk
4375
4376                     # skip non hotpluggable value
4377                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4378                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4379                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4380                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4381                         die "skip\n";
4382                     }
4383
4384                     # apply throttle
4385                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4386                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4387                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4388                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4389                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4390                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4391                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4392                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4393                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4394                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4395                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4396                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4397                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4398                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4399                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4400                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4401                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4402                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4403
4404                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4405                                                    ($drive->{mbps} || 0)*1024*1024,
4406                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4407                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4408                                                    $drive->{iops} || 0,
4409                                                    $drive->{iops_rd} || 0,
4410                                                    $drive->{iops_wr} || 0,
4411                                                    ($drive->{mbps_max} || 0)*1024*1024,
4412                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4413                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4414                                                    $drive->{iops_max} || 0,
4415                                                    $drive->{iops_rd_max} || 0,
4416                                                    $drive->{iops_wr_max} || 0,
4417                                                    $drive->{bps_max_length} || 1,
4418                                                    $drive->{bps_rd_max_length} || 1,
4419                                                    $drive->{bps_wr_max_length} || 1,
4420                                                    $drive->{iops_max_length} || 1,
4421                                                    $drive->{iops_rd_max_length} || 1,
4422                                                    $drive->{iops_wr_max_length} || 1);
4423
4424                     }
4425
4426                     return 1;
4427                 }
4428
4429             } else { # cdrom
4430
4431                 if ($drive->{file} eq 'none') {
4432                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4433                 } else {
4434                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4435                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4436                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4437                 }
4438
4439                 return 1;
4440             }
4441         }
4442     }
4443
4444     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4445     # hotplug new disks
4446     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4447     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4448 }
4449
4450 sub vm_start {
4451     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4452         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4453
4454     PVE::QemuConfig->lock_config($vmid, sub {
4455         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4456
4457         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4458
4459         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4460
4461         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4462
4463         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4464             vmconfig_apply_pending($vmid, $conf, $storecfg);
4465             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4466         }
4467
4468         my $defaults = load_defaults();
4469
4470         # set environment variable useful inside network script
4471         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4472
4473         my $local_volumes = {};
4474
4475         if( $targetstorage ){
4476
4477             foreach_drive($conf, sub {
4478                 my ($ds, $drive) = @_;
4479
4480                 return if drive_is_cdrom($drive);
4481
4482                 my $volid = $drive->{file};
4483
4484                 return if !$volid;
4485
4486                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4487
4488                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4489                 return if $scfg->{shared};
4490                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4491             });
4492
4493             my $format = undef;
4494
4495             foreach my $opt (sort keys %$local_volumes) {
4496
4497                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4498                 my $drive = parse_drive($opt, $conf->{$opt});
4499
4500                 #if remote storage is specified, use default format
4501                 if ($targetstorage && $targetstorage ne "1") {
4502                     $storeid = $targetstorage;
4503                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4504                     $format = $defFormat;
4505                 } else {
4506                     #else we use same format than original
4507                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4508                     $format = qemu_img_format($scfg, $volid);
4509                 }
4510
4511                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4512                 my $newdrive = $drive;
4513                 $newdrive->{format} = $format;
4514                 $newdrive->{file} = $newvolid;
4515                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4516                 $local_volumes->{$opt} = $drivestr;
4517                 #pass drive to conf for command line
4518                 $conf->{$opt} = $drivestr;
4519             }
4520         }
4521
4522         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4523
4524         my $migrate_port = 0;
4525         my $migrate_uri;
4526         if ($statefile) {
4527             if ($statefile eq 'tcp') {
4528                 my $localip = "localhost";
4529                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4530                 my $nodename = PVE::INotify::nodename();
4531
4532                 if (!defined($migration_type)) {
4533                     if (defined($datacenterconf->{migration}->{type})) {
4534                         $migration_type = $datacenterconf->{migration}->{type};
4535                     } else {
4536                         $migration_type = 'secure';
4537                     }
4538                 }
4539
4540                 if ($migration_type eq 'insecure') {
4541                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4542                     if ($migrate_network_addr) {
4543                         $localip = $migrate_network_addr;
4544                     } else {
4545                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4546                     }
4547
4548                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4549                 }
4550
4551                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4552                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4553                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4554                 push @$cmd, '-incoming', $migrate_uri;
4555                 push @$cmd, '-S';
4556
4557             } elsif ($statefile eq 'unix') {
4558                 # should be default for secure migrations as a ssh TCP forward
4559                 # tunnel is not deterministic reliable ready and fails regurarly
4560                 # to set up in time, so use UNIX socket forwards
4561                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4562                 unlink $socket_addr;
4563
4564                 $migrate_uri = "unix:$socket_addr";
4565
4566                 push @$cmd, '-incoming', $migrate_uri;
4567                 push @$cmd, '-S';
4568
4569             } else {
4570                 push @$cmd, '-loadstate', $statefile;
4571             }
4572         } elsif ($paused) {
4573             push @$cmd, '-S';
4574         }
4575
4576         # host pci devices
4577         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4578           my $d = parse_hostpci($conf->{"hostpci$i"});
4579           next if !$d;
4580           my $pcidevices = $d->{pciid};
4581           foreach my $pcidevice (@$pcidevices) {
4582                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4583
4584                 my $info = pci_device_info("0000:$pciid");
4585                 die "IOMMU not present\n" if !check_iommu_support();
4586                 die "no pci device info for device '$pciid'\n" if !$info;
4587                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4588                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4589           }
4590         }
4591
4592         PVE::Storage::activate_volumes($storecfg, $vollist);
4593
4594         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4595             my $cmd = [];
4596             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4597             eval  { run_command($cmd); };
4598         }
4599
4600         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4601                                                   : $defaults->{cpuunits};
4602
4603         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4604
4605         my %properties = (
4606             Slice => 'qemu.slice',
4607             KillMode => 'none',
4608             CPUShares => $cpuunits
4609         );
4610
4611         if (my $cpulimit = $conf->{cpulimit}) {
4612             $properties{CPUQuota} = int($cpulimit * 100);
4613         }
4614         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4615
4616         if ($conf->{hugepages}) {
4617
4618             my $code = sub {
4619                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4620                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4621
4622                 PVE::QemuServer::Memory::hugepages_mount();
4623                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4624
4625                 eval  {
4626                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4627                     run_command($cmd, %run_params);
4628                 };
4629
4630                 if (my $err = $@) {
4631                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4632                     die $err;
4633                 }
4634
4635                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4636             };
4637             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4638
4639         } else {
4640             eval  {
4641                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4642                 run_command($cmd, %run_params);
4643             };
4644         }
4645
4646         if (my $err = $@) {
4647             # deactivate volumes if start fails
4648             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4649             die "start failed: $err";
4650         }
4651
4652         print "migration listens on $migrate_uri\n" if $migrate_uri;
4653
4654         if ($statefile && $statefile ne 'tcp')  {
4655             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4656             warn $@ if $@;
4657         }
4658
4659         #start nbd server for storage migration
4660         if ($targetstorage) {
4661
4662             my $nodename = PVE::INotify::nodename();
4663             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4664             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4665             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4666             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4667
4668             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4669
4670             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4671
4672             foreach my $opt (sort keys %$local_volumes) {
4673                 my $volid = $local_volumes->{$opt};
4674                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4675                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4676                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4677             }
4678         }
4679
4680         if ($migratedfrom) {
4681
4682             eval {
4683                 set_migration_caps($vmid);
4684             };
4685             warn $@ if $@;
4686
4687             if ($spice_port) {
4688                 print "spice listens on port $spice_port\n";
4689                 if ($spice_ticket) {
4690                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4691                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4692                 }
4693             }
4694
4695         } else {
4696
4697             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4698                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4699                     if $conf->{balloon};
4700             }
4701
4702             foreach my $opt (keys %$conf) {
4703                 next if $opt !~  m/^net\d+$/;
4704                 my $nicconf = parse_net($conf->{$opt});
4705                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4706             }
4707         }
4708
4709         vm_mon_cmd_nocheck($vmid, 'qom-set',
4710                     path => "machine/peripheral/balloon0",
4711                     property => "guest-stats-polling-interval",
4712                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4713
4714     });
4715 }
4716
4717 sub vm_mon_cmd {
4718     my ($vmid, $execute, %params) = @_;
4719
4720     my $cmd = { execute => $execute, arguments => \%params };
4721     vm_qmp_command($vmid, $cmd);
4722 }
4723
4724 sub vm_mon_cmd_nocheck {
4725     my ($vmid, $execute, %params) = @_;
4726
4727     my $cmd = { execute => $execute, arguments => \%params };
4728     vm_qmp_command($vmid, $cmd, 1);
4729 }
4730
4731 sub vm_qmp_command {
4732     my ($vmid, $cmd, $nocheck) = @_;
4733
4734     my $res;
4735
4736     my $timeout;
4737     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4738         $timeout = $cmd->{arguments}->{timeout};
4739         delete $cmd->{arguments}->{timeout};
4740     }
4741
4742     eval {
4743         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4744         my $sname = qmp_socket($vmid);
4745         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4746             my $qmpclient = PVE::QMPClient->new();
4747
4748             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4749         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4750             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4751                 if scalar(%{$cmd->{arguments}});
4752             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4753         } else {
4754             die "unable to open monitor socket\n";
4755         }
4756     };
4757     if (my $err = $@) {
4758         syslog("err", "VM $vmid qmp command failed - $err");
4759         die $err;
4760     }
4761
4762     return $res;
4763 }
4764
4765 sub vm_human_monitor_command {
4766     my ($vmid, $cmdline) = @_;
4767
4768     my $res;
4769
4770     my $cmd = {
4771         execute => 'human-monitor-command',
4772         arguments => { 'command-line' => $cmdline},
4773     };
4774
4775     return vm_qmp_command($vmid, $cmd);
4776 }
4777
4778 sub vm_commandline {
4779     my ($storecfg, $vmid) = @_;
4780
4781     my $conf = PVE::QemuConfig->load_config($vmid);
4782
4783     my $defaults = load_defaults();
4784
4785     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4786
4787     return PVE::Tools::cmd2string($cmd);
4788 }
4789
4790 sub vm_reset {
4791     my ($vmid, $skiplock) = @_;
4792
4793     PVE::QemuConfig->lock_config($vmid, sub {
4794
4795         my $conf = PVE::QemuConfig->load_config($vmid);
4796
4797         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4798
4799         vm_mon_cmd($vmid, "system_reset");
4800     });
4801 }
4802
4803 sub get_vm_volumes {
4804     my ($conf) = @_;
4805
4806     my $vollist = [];
4807     foreach_volid($conf, sub {
4808         my ($volid, $is_cdrom) = @_;
4809
4810         return if $volid =~ m|^/|;
4811
4812         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4813         return if !$sid;
4814
4815         push @$vollist, $volid;
4816     });
4817
4818     return $vollist;
4819 }
4820
4821 sub vm_stop_cleanup {
4822     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4823
4824     eval {
4825
4826         if (!$keepActive) {
4827             my $vollist = get_vm_volumes($conf);
4828             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4829         }
4830
4831         foreach my $ext (qw(mon qmp pid vnc qga)) {
4832             unlink "/var/run/qemu-server/${vmid}.$ext";
4833         }
4834
4835         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4836     };
4837     warn $@ if $@; # avoid errors - just warn
4838 }
4839
4840 # Note: use $nockeck to skip tests if VM configuration file exists.
4841 # We need that when migration VMs to other nodes (files already moved)
4842 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4843 sub vm_stop {
4844     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4845
4846     $force = 1 if !defined($force) && !$shutdown;
4847
4848     if ($migratedfrom){
4849         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4850         kill 15, $pid if $pid;
4851         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4852         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4853         return;
4854     }
4855
4856     PVE::QemuConfig->lock_config($vmid, sub {
4857
4858         my $pid = check_running($vmid, $nocheck);
4859         return if !$pid;
4860
4861         my $conf;
4862         if (!$nocheck) {
4863             $conf = PVE::QemuConfig->load_config($vmid);
4864             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4865             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4866                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4867                 $timeout = $opts->{down} if $opts->{down};
4868             }
4869         }
4870
4871         $timeout = 60 if !defined($timeout);
4872
4873         eval {
4874             if ($shutdown) {
4875                 if (defined($conf) && $conf->{agent}) {
4876                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4877                 } else {
4878                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4879                 }
4880             } else {
4881                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4882             }
4883         };
4884         my $err = $@;
4885
4886         if (!$err) {
4887             my $count = 0;
4888             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4889                 $count++;
4890                 sleep 1;
4891             }
4892
4893             if ($count >= $timeout) {
4894                 if ($force) {
4895                     warn "VM still running - terminating now with SIGTERM\n";
4896                     kill 15, $pid;
4897                 } else {
4898                     die "VM quit/powerdown failed - got timeout\n";
4899                 }
4900             } else {
4901                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4902                 return;
4903             }
4904         } else {
4905             if ($force) {
4906                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4907                 kill 15, $pid;
4908             } else {
4909                 die "VM quit/powerdown failed\n";
4910             }
4911         }
4912
4913         # wait again
4914         $timeout = 10;
4915
4916         my $count = 0;
4917         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4918             $count++;
4919             sleep 1;
4920         }
4921
4922         if ($count >= $timeout) {
4923             warn "VM still running - terminating now with SIGKILL\n";
4924             kill 9, $pid;
4925             sleep 1;
4926         }
4927
4928         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4929    });
4930 }
4931
4932 sub vm_suspend {
4933     my ($vmid, $skiplock) = @_;
4934
4935     PVE::QemuConfig->lock_config($vmid, sub {
4936
4937         my $conf = PVE::QemuConfig->load_config($vmid);
4938
4939         PVE::QemuConfig->check_lock($conf)
4940             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4941
4942         vm_mon_cmd($vmid, "stop");
4943     });
4944 }
4945
4946 sub vm_resume {
4947     my ($vmid, $skiplock, $nocheck) = @_;
4948
4949     PVE::QemuConfig->lock_config($vmid, sub {
4950
4951         if (!$nocheck) {
4952
4953             my $conf = PVE::QemuConfig->load_config($vmid);
4954
4955             PVE::QemuConfig->check_lock($conf)
4956                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4957
4958             vm_mon_cmd($vmid, "cont");
4959
4960         } else {
4961             vm_mon_cmd_nocheck($vmid, "cont");
4962         }
4963     });
4964 }
4965
4966 sub vm_sendkey {
4967     my ($vmid, $skiplock, $key) = @_;
4968
4969     PVE::QemuConfig->lock_config($vmid, sub {
4970
4971         my $conf = PVE::QemuConfig->load_config($vmid);
4972
4973         # there is no qmp command, so we use the human monitor command
4974         vm_human_monitor_command($vmid, "sendkey $key");
4975     });
4976 }
4977
4978 sub vm_destroy {
4979     my ($storecfg, $vmid, $skiplock) = @_;
4980
4981     PVE::QemuConfig->lock_config($vmid, sub {
4982
4983         my $conf = PVE::QemuConfig->load_config($vmid);
4984
4985         if (!check_running($vmid)) {
4986             destroy_vm($storecfg, $vmid, undef, $skiplock);
4987         } else {
4988             die "VM $vmid is running - destroy failed\n";
4989         }
4990     });
4991 }
4992
4993 # pci helpers
4994
4995 sub file_write {
4996     my ($filename, $buf) = @_;
4997
4998     my $fh = IO::File->new($filename, "w");
4999     return undef if !$fh;
5000
5001     my $res = print $fh $buf;
5002
5003     $fh->close();
5004
5005     return $res;
5006 }
5007
5008 sub pci_device_info {
5009     my ($name) = @_;
5010
5011     my $res;
5012
5013     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5014     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5015
5016     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5017     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5018
5019     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5020     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5021
5022     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5023     return undef if !defined($product) || $product !~ s/^0x//;
5024
5025     $res = {
5026         name => $name,
5027         vendor => $vendor,
5028         product => $product,
5029         domain => $domain,
5030         bus => $bus,
5031         slot => $slot,
5032         func => $func,
5033         irq => $irq,
5034         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5035     };
5036
5037     return $res;
5038 }
5039
5040 sub pci_dev_reset {
5041     my ($dev) = @_;
5042
5043     my $name = $dev->{name};
5044
5045     my $fn = "$pcisysfs/devices/$name/reset";
5046
5047     return file_write($fn, "1");
5048 }
5049
5050 sub pci_dev_bind_to_vfio {
5051     my ($dev) = @_;
5052
5053     my $name = $dev->{name};
5054
5055     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5056
5057     if (!-d $vfio_basedir) {
5058         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5059     }
5060     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5061
5062     my $testdir = "$vfio_basedir/$name";
5063     return 1 if -d $testdir;
5064
5065     my $data = "$dev->{vendor} $dev->{product}";
5066     return undef if !file_write("$vfio_basedir/new_id", $data);
5067
5068     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5069     if (!file_write($fn, $name)) {
5070         return undef if -f $fn;
5071     }
5072
5073     $fn = "$vfio_basedir/bind";
5074     if (! -d $testdir) {
5075         return undef if !file_write($fn, $name);
5076     }
5077
5078     return -d $testdir;
5079 }
5080
5081 sub pci_dev_group_bind_to_vfio {
5082     my ($pciid) = @_;
5083
5084     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5085
5086     if (!-d $vfio_basedir) {
5087         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5088     }
5089     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5090
5091     # get IOMMU group devices
5092     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5093       my @devs = grep /^0000:/, readdir($D);
5094     closedir($D);
5095
5096     foreach my $pciid (@devs) {
5097         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5098
5099         # pci bridges, switches or root ports are not supported
5100         # they have a pci_bus subdirectory so skip them
5101         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5102
5103         my $info = pci_device_info($1);
5104         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5105     }
5106
5107     return 1;
5108 }
5109
5110 # vzdump restore implementaion
5111
5112 sub tar_archive_read_firstfile {
5113     my $archive = shift;
5114
5115     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5116
5117     # try to detect archive type first
5118     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5119         die "unable to open file '$archive'\n";
5120     my $firstfile = <$fh>;
5121     kill 15, $pid;
5122     close $fh;
5123
5124     die "ERROR: archive contaions no data\n" if !$firstfile;
5125     chomp $firstfile;
5126
5127     return $firstfile;
5128 }
5129
5130 sub tar_restore_cleanup {
5131     my ($storecfg, $statfile) = @_;
5132
5133     print STDERR "starting cleanup\n";
5134
5135     if (my $fd = IO::File->new($statfile, "r")) {
5136         while (defined(my $line = <$fd>)) {
5137             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5138                 my $volid = $2;
5139                 eval {
5140                     if ($volid =~ m|^/|) {
5141                         unlink $volid || die 'unlink failed\n';
5142                     } else {
5143                         PVE::Storage::vdisk_free($storecfg, $volid);
5144                     }
5145                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5146                 };
5147                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5148             } else {
5149                 print STDERR "unable to parse line in statfile - $line";
5150             }
5151         }
5152         $fd->close();
5153     }
5154 }
5155
5156 sub restore_archive {
5157     my ($archive, $vmid, $user, $opts) = @_;
5158
5159     my $format = $opts->{format};
5160     my $comp;
5161
5162     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5163         $format = 'tar' if !$format;
5164         $comp = 'gzip';
5165     } elsif ($archive =~ m/\.tar$/) {
5166         $format = 'tar' if !$format;
5167     } elsif ($archive =~ m/.tar.lzo$/) {
5168         $format = 'tar' if !$format;
5169         $comp = 'lzop';
5170     } elsif ($archive =~ m/\.vma$/) {
5171         $format = 'vma' if !$format;
5172     } elsif ($archive =~ m/\.vma\.gz$/) {
5173         $format = 'vma' if !$format;
5174         $comp = 'gzip';
5175     } elsif ($archive =~ m/\.vma\.lzo$/) {
5176         $format = 'vma' if !$format;
5177         $comp = 'lzop';
5178     } else {
5179         $format = 'vma' if !$format; # default
5180     }
5181
5182     # try to detect archive format
5183     if ($format eq 'tar') {
5184         return restore_tar_archive($archive, $vmid, $user, $opts);
5185     } else {
5186         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5187     }
5188 }
5189
5190 sub restore_update_config_line {
5191     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5192
5193     return if $line =~ m/^\#qmdump\#/;
5194     return if $line =~ m/^\#vzdump\#/;
5195     return if $line =~ m/^lock:/;
5196     return if $line =~ m/^unused\d+:/;
5197     return if $line =~ m/^parent:/;
5198     return if $line =~ m/^template:/; # restored VM is never a template
5199
5200     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5201     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5202         # try to convert old 1.X settings
5203         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5204         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5205             my ($model, $macaddr) = split(/\=/, $devconfig);
5206             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5207             my $net = {
5208                 model => $model,
5209                 bridge => "vmbr$ind",
5210                 macaddr => $macaddr,
5211             };
5212             my $netstr = print_net($net);
5213
5214             print $outfd "net$cookie->{netcount}: $netstr\n";
5215             $cookie->{netcount}++;
5216         }
5217     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5218         my ($id, $netstr) = ($1, $2);
5219         my $net = parse_net($netstr);
5220         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5221         $netstr = print_net($net);
5222         print $outfd "$id: $netstr\n";
5223     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5224         my $virtdev = $1;
5225         my $value = $3;
5226         my $di = parse_drive($virtdev, $value);
5227         if (defined($di->{backup}) && !$di->{backup}) {
5228             print $outfd "#$line";
5229         } elsif ($map->{$virtdev}) {
5230             delete $di->{format}; # format can change on restore
5231             $di->{file} = $map->{$virtdev};
5232             $value = print_drive($vmid, $di);
5233             print $outfd "$virtdev: $value\n";
5234         } else {
5235             print $outfd $line;
5236         }
5237     } else {
5238         print $outfd $line;
5239     }
5240 }
5241
5242 sub scan_volids {
5243     my ($cfg, $vmid) = @_;
5244
5245     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5246
5247     my $volid_hash = {};
5248     foreach my $storeid (keys %$info) {
5249         foreach my $item (@{$info->{$storeid}}) {
5250             next if !($item->{volid} && $item->{size});
5251             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5252             $volid_hash->{$item->{volid}} = $item;
5253         }
5254     }
5255
5256     return $volid_hash;
5257 }
5258
5259 sub is_volume_in_use {
5260     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5261
5262     my $path = PVE::Storage::path($storecfg, $volid);
5263
5264     my $scan_config = sub {
5265         my ($cref, $snapname) = @_;
5266
5267         foreach my $key (keys %$cref) {
5268             my $value = $cref->{$key};
5269             if (is_valid_drivename($key)) {
5270                 next if $skip_drive && $key eq $skip_drive;
5271                 my $drive = parse_drive($key, $value);
5272                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5273                 return 1 if $volid eq $drive->{file};
5274                 if ($drive->{file} =~ m!^/!) {
5275                     return 1 if $drive->{file} eq $path;
5276                 } else {
5277                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5278                     next if !$storeid;
5279                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5280                     next if !$scfg;
5281                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5282                 }
5283             }
5284         }
5285
5286         return 0;
5287     };
5288
5289     return 1 if &$scan_config($conf);
5290
5291     undef $skip_drive;
5292
5293     foreach my $snapname (keys %{$conf->{snapshots}}) {
5294         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5295     }
5296
5297     return 0;
5298 }
5299
5300 sub update_disksize {
5301     my ($vmid, $conf, $volid_hash) = @_;
5302
5303     my $changes;
5304
5305     my $used = {};
5306
5307     # Note: it is allowed to define multiple storages with same path (alias), so
5308     # we need to check both 'volid' and real 'path' (two different volid can point
5309     # to the same path).
5310
5311     my $usedpath = {};
5312
5313     # update size info
5314     foreach my $opt (keys %$conf) {
5315         if (is_valid_drivename($opt)) {
5316             my $drive = parse_drive($opt, $conf->{$opt});
5317             my $volid = $drive->{file};
5318             next if !$volid;
5319
5320             $used->{$volid} = 1;
5321             if ($volid_hash->{$volid} &&
5322                 (my $path = $volid_hash->{$volid}->{path})) {
5323                 $usedpath->{$path} = 1;
5324             }
5325
5326             next if drive_is_cdrom($drive);
5327             next if !$volid_hash->{$volid};
5328
5329             $drive->{size} = $volid_hash->{$volid}->{size};
5330             my $new = print_drive($vmid, $drive);
5331             if ($new ne $conf->{$opt}) {
5332                 $changes = 1;
5333                 $conf->{$opt} = $new;
5334             }
5335         }
5336     }
5337
5338     # remove 'unusedX' entry if volume is used
5339     foreach my $opt (keys %$conf) {
5340         next if $opt !~ m/^unused\d+$/;
5341         my $volid = $conf->{$opt};
5342         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5343         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5344             $changes = 1;
5345             delete $conf->{$opt};
5346         }
5347     }
5348
5349     foreach my $volid (sort keys %$volid_hash) {
5350         next if $volid =~ m/vm-$vmid-state-/;
5351         next if $used->{$volid};
5352         my $path = $volid_hash->{$volid}->{path};
5353         next if !$path; # just to be sure
5354         next if $usedpath->{$path};
5355         $changes = 1;
5356         PVE::QemuConfig->add_unused_volume($conf, $volid);
5357         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5358     }
5359
5360     return $changes;
5361 }
5362
5363 sub rescan {
5364     my ($vmid, $nolock) = @_;
5365
5366     my $cfg = PVE::Storage::config();
5367
5368     my $volid_hash = scan_volids($cfg, $vmid);
5369
5370     my $updatefn =  sub {
5371         my ($vmid) = @_;
5372
5373         my $conf = PVE::QemuConfig->load_config($vmid);
5374
5375         PVE::QemuConfig->check_lock($conf);
5376
5377         my $vm_volids = {};
5378         foreach my $volid (keys %$volid_hash) {
5379             my $info = $volid_hash->{$volid};
5380             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5381         }
5382
5383         my $changes = update_disksize($vmid, $conf, $vm_volids);
5384
5385         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5386     };
5387
5388     if (defined($vmid)) {
5389         if ($nolock) {
5390             &$updatefn($vmid);
5391         } else {
5392             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5393         }
5394     } else {
5395         my $vmlist = config_list();
5396         foreach my $vmid (keys %$vmlist) {
5397             if ($nolock) {
5398                 &$updatefn($vmid);
5399             } else {
5400                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5401             }
5402         }
5403     }
5404 }
5405
5406 sub restore_vma_archive {
5407     my ($archive, $vmid, $user, $opts, $comp) = @_;
5408
5409     my $input = $archive eq '-' ? "<&STDIN" : undef;
5410     my $readfrom = $archive;
5411
5412     my $uncomp = '';
5413     if ($comp) {
5414         $readfrom = '-';
5415         my $qarchive = PVE::Tools::shellquote($archive);
5416         if ($comp eq 'gzip') {
5417             $uncomp = "zcat $qarchive|";
5418         } elsif ($comp eq 'lzop') {
5419             $uncomp = "lzop -d -c $qarchive|";
5420         } else {
5421             die "unknown compression method '$comp'\n";
5422         }
5423
5424     }
5425
5426     my $tmpdir = "/var/tmp/vzdumptmp$$";
5427     rmtree $tmpdir;
5428
5429     # disable interrupts (always do cleanups)
5430     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5431         warn "got interrupt - ignored\n";
5432     };
5433
5434     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5435     POSIX::mkfifo($mapfifo, 0600);
5436     my $fifofh;
5437
5438     my $openfifo = sub {
5439         open($fifofh, '>', $mapfifo) || die $!;
5440     };
5441
5442     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5443
5444     my $oldtimeout;
5445     my $timeout = 5;
5446
5447     my $devinfo = {};
5448
5449     my $rpcenv = PVE::RPCEnvironment::get();
5450
5451     my $conffile = PVE::QemuConfig->config_file($vmid);
5452     my $tmpfn = "$conffile.$$.tmp";
5453
5454     # Note: $oldconf is undef if VM does not exists
5455     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5456     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5457
5458     my $print_devmap = sub {
5459         my $virtdev_hash = {};
5460
5461         my $cfgfn = "$tmpdir/qemu-server.conf";
5462
5463         # we can read the config - that is already extracted
5464         my $fh = IO::File->new($cfgfn, "r") ||
5465             "unable to read qemu-server.conf - $!\n";
5466
5467         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5468         if (-f $fwcfgfn) {
5469             my $pve_firewall_dir = '/etc/pve/firewall';
5470             mkdir $pve_firewall_dir; # make sure the dir exists
5471             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5472         }
5473
5474         while (defined(my $line = <$fh>)) {
5475             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5476                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5477                 die "archive does not contain data for drive '$virtdev'\n"
5478                     if !$devinfo->{$devname};
5479                 if (defined($opts->{storage})) {
5480                     $storeid = $opts->{storage} || 'local';
5481                 } elsif (!$storeid) {
5482                     $storeid = 'local';
5483                 }
5484                 $format = 'raw' if !$format;
5485                 $devinfo->{$devname}->{devname} = $devname;
5486                 $devinfo->{$devname}->{virtdev} = $virtdev;
5487                 $devinfo->{$devname}->{format} = $format;
5488                 $devinfo->{$devname}->{storeid} = $storeid;
5489
5490                 # check permission on storage
5491                 my $pool = $opts->{pool}; # todo: do we need that?
5492                 if ($user ne 'root@pam') {
5493                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5494                 }
5495
5496                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5497             }
5498         }
5499
5500         foreach my $devname (keys %$devinfo) {
5501             die "found no device mapping information for device '$devname'\n"
5502                 if !$devinfo->{$devname}->{virtdev};
5503         }
5504
5505         my $cfg = PVE::Storage::config();
5506
5507         # create empty/temp config
5508         if ($oldconf) {
5509             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5510             foreach_drive($oldconf, sub {
5511                 my ($ds, $drive) = @_;
5512
5513                 return if drive_is_cdrom($drive);
5514
5515                 my $volid = $drive->{file};
5516
5517                 return if !$volid || $volid =~ m|^/|;
5518
5519                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5520                 return if !$path || !$owner || ($owner != $vmid);
5521
5522                 # Note: only delete disk we want to restore
5523                 # other volumes will become unused
5524                 if ($virtdev_hash->{$ds}) {
5525                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5526                     if (my $err = $@) {
5527                         warn $err;
5528                     }
5529                 }
5530             });
5531
5532             # delete vmstate files
5533             # since after the restore we have no snapshots anymore
5534             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5535                 my $snap = $oldconf->{snapshots}->{$snapname};
5536                 if ($snap->{vmstate}) {
5537                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5538                     if (my $err = $@) {
5539                         warn $err;
5540                     }
5541                 }
5542             }
5543         }
5544
5545         my $map = {};
5546         foreach my $virtdev (sort keys %$virtdev_hash) {
5547             my $d = $virtdev_hash->{$virtdev};
5548             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5549             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5550
5551             # test if requested format is supported
5552             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5553             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5554             $d->{format} = $defFormat if !$supported;
5555
5556             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5557                                                   $d->{format}, undef, $alloc_size);
5558             print STDERR "new volume ID is '$volid'\n";
5559             $d->{volid} = $volid;
5560             my $path = PVE::Storage::path($cfg, $volid);
5561
5562             PVE::Storage::activate_volumes($cfg,[$volid]);
5563
5564             my $write_zeros = 1;
5565             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5566                 $write_zeros = 0;
5567             }
5568
5569             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5570
5571             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5572             $map->{$virtdev} = $volid;
5573         }
5574
5575         $fh->seek(0, 0) || die "seek failed - $!\n";
5576
5577         my $outfd = new IO::File ($tmpfn, "w") ||
5578             die "unable to write config for VM $vmid\n";
5579
5580         my $cookie = { netcount => 0 };
5581         while (defined(my $line = <$fh>)) {
5582             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5583         }
5584
5585         $fh->close();
5586         $outfd->close();
5587     };
5588
5589     eval {
5590         # enable interrupts
5591         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5592             die "interrupted by signal\n";
5593         };
5594         local $SIG{ALRM} = sub { die "got timeout\n"; };
5595
5596         $oldtimeout = alarm($timeout);
5597
5598         my $parser = sub {
5599             my $line = shift;
5600
5601             print "$line\n";
5602
5603             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5604                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5605                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5606             } elsif ($line =~ m/^CTIME: /) {
5607                 # we correctly received the vma config, so we can disable
5608                 # the timeout now for disk allocation (set to 10 minutes, so
5609                 # that we always timeout if something goes wrong)
5610                 alarm(600);
5611                 &$print_devmap();
5612                 print $fifofh "done\n";
5613                 my $tmp = $oldtimeout || 0;
5614                 $oldtimeout = undef;
5615                 alarm($tmp);
5616                 close($fifofh);
5617             }
5618         };
5619
5620         print "restore vma archive: $cmd\n";
5621         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5622     };
5623     my $err = $@;
5624
5625     alarm($oldtimeout) if $oldtimeout;
5626
5627     my $vollist = [];
5628     foreach my $devname (keys %$devinfo) {
5629         my $volid = $devinfo->{$devname}->{volid};
5630         push @$vollist, $volid if $volid;
5631     }
5632
5633     my $cfg = PVE::Storage::config();
5634     PVE::Storage::deactivate_volumes($cfg, $vollist);
5635
5636     unlink $mapfifo;
5637
5638     if ($err) {
5639         rmtree $tmpdir;
5640         unlink $tmpfn;
5641
5642         foreach my $devname (keys %$devinfo) {
5643             my $volid = $devinfo->{$devname}->{volid};
5644             next if !$volid;
5645             eval {
5646                 if ($volid =~ m|^/|) {
5647                     unlink $volid || die 'unlink failed\n';
5648                 } else {
5649                     PVE::Storage::vdisk_free($cfg, $volid);
5650                 }
5651                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5652             };
5653             print STDERR "unable to cleanup '$volid' - $@" if $@;
5654         }
5655         die $err;
5656     }
5657
5658     rmtree $tmpdir;
5659
5660     rename($tmpfn, $conffile) ||
5661         die "unable to commit configuration file '$conffile'\n";
5662
5663     PVE::Cluster::cfs_update(); # make sure we read new file
5664
5665     eval { rescan($vmid, 1); };
5666     warn $@ if $@;
5667 }
5668
5669 sub restore_tar_archive {
5670     my ($archive, $vmid, $user, $opts) = @_;
5671
5672     if ($archive ne '-') {
5673         my $firstfile = tar_archive_read_firstfile($archive);
5674         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5675             if $firstfile ne 'qemu-server.conf';
5676     }
5677
5678     my $storecfg = PVE::Storage::config();
5679
5680     # destroy existing data - keep empty config
5681     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5682     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5683
5684     my $tocmd = "/usr/lib/qemu-server/qmextract";
5685
5686     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5687     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5688     $tocmd .= ' --prealloc' if $opts->{prealloc};
5689     $tocmd .= ' --info' if $opts->{info};
5690
5691     # tar option "xf" does not autodetect compression when read from STDIN,
5692     # so we pipe to zcat
5693     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5694         PVE::Tools::shellquote("--to-command=$tocmd");
5695
5696     my $tmpdir = "/var/tmp/vzdumptmp$$";
5697     mkpath $tmpdir;
5698
5699     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5700     local $ENV{VZDUMP_VMID} = $vmid;
5701     local $ENV{VZDUMP_USER} = $user;
5702
5703     my $conffile = PVE::QemuConfig->config_file($vmid);
5704     my $tmpfn = "$conffile.$$.tmp";
5705
5706     # disable interrupts (always do cleanups)
5707     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5708         print STDERR "got interrupt - ignored\n";
5709     };
5710
5711     eval {
5712         # enable interrupts
5713         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5714             die "interrupted by signal\n";
5715         };
5716
5717         if ($archive eq '-') {
5718             print "extracting archive from STDIN\n";
5719             run_command($cmd, input => "<&STDIN");
5720         } else {
5721             print "extracting archive '$archive'\n";
5722             run_command($cmd);
5723         }
5724
5725         return if $opts->{info};
5726
5727         # read new mapping
5728         my $map = {};
5729         my $statfile = "$tmpdir/qmrestore.stat";
5730         if (my $fd = IO::File->new($statfile, "r")) {
5731             while (defined (my $line = <$fd>)) {
5732                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5733                     $map->{$1} = $2 if $1;
5734                 } else {
5735                     print STDERR "unable to parse line in statfile - $line\n";
5736                 }
5737             }
5738             $fd->close();
5739         }
5740
5741         my $confsrc = "$tmpdir/qemu-server.conf";
5742
5743         my $srcfd = new IO::File($confsrc, "r") ||
5744             die "unable to open file '$confsrc'\n";
5745
5746         my $outfd = new IO::File ($tmpfn, "w") ||
5747             die "unable to write config for VM $vmid\n";
5748
5749         my $cookie = { netcount => 0 };
5750         while (defined (my $line = <$srcfd>)) {
5751             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5752         }
5753
5754         $srcfd->close();
5755         $outfd->close();
5756     };
5757     my $err = $@;
5758
5759     if ($err) {
5760
5761         unlink $tmpfn;
5762
5763         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5764
5765         die $err;
5766     }
5767
5768     rmtree $tmpdir;
5769
5770     rename $tmpfn, $conffile ||
5771         die "unable to commit configuration file '$conffile'\n";
5772
5773     PVE::Cluster::cfs_update(); # make sure we read new file
5774
5775     eval { rescan($vmid, 1); };
5776     warn $@ if $@;
5777 };
5778
5779 sub foreach_writable_storage {
5780     my ($conf, $func) = @_;
5781
5782     my $sidhash = {};
5783
5784     foreach my $ds (keys %$conf) {
5785         next if !is_valid_drivename($ds);
5786
5787         my $drive = parse_drive($ds, $conf->{$ds});
5788         next if !$drive;
5789         next if drive_is_cdrom($drive);
5790
5791         my $volid = $drive->{file};
5792
5793         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5794         $sidhash->{$sid} = $sid if $sid;
5795     }
5796
5797     foreach my $sid (sort keys %$sidhash) {
5798         &$func($sid);
5799     }
5800 }
5801
5802 sub do_snapshots_with_qemu {
5803     my ($storecfg, $volid) = @_;
5804
5805     my $storage_name = PVE::Storage::parse_volume_id($volid);
5806
5807     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5808         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5809         return 1;
5810     }
5811
5812     if ($volid =~ m/\.(qcow2|qed)$/){
5813         return 1;
5814     }
5815
5816     return undef;
5817 }
5818
5819 sub qga_check_running {
5820     my ($vmid) = @_;
5821
5822     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5823     if ($@) {
5824         warn "Qemu Guest Agent are not running - $@";
5825         return 0;
5826     }
5827     return 1;
5828 }
5829
5830 sub template_create {
5831     my ($vmid, $conf, $disk) = @_;
5832
5833     my $storecfg = PVE::Storage::config();
5834
5835     foreach_drive($conf, sub {
5836         my ($ds, $drive) = @_;
5837
5838         return if drive_is_cdrom($drive);
5839         return if $disk && $ds ne $disk;
5840
5841         my $volid = $drive->{file};
5842         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5843
5844         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5845         $drive->{file} = $voliddst;
5846         $conf->{$ds} = print_drive($vmid, $drive);
5847         PVE::QemuConfig->write_config($vmid, $conf);
5848     });
5849 }
5850
5851 sub qemu_img_convert {
5852     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5853
5854     my $storecfg = PVE::Storage::config();
5855     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5856     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5857
5858     if ($src_storeid && $dst_storeid) {
5859
5860         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5861
5862         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5863         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5864
5865         my $src_format = qemu_img_format($src_scfg, $src_volname);
5866         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5867
5868         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5869         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5870
5871         my $cmd = [];
5872         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5873         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5874         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5875         if ($is_zero_initialized) {
5876             push @$cmd, "zeroinit:$dst_path";
5877         } else {
5878             push @$cmd, $dst_path;
5879         }
5880
5881         my $parser = sub {
5882             my $line = shift;
5883             if($line =~ m/\((\S+)\/100\%\)/){
5884                 my $percent = $1;
5885                 my $transferred = int($size * $percent / 100);
5886                 my $remaining = $size - $transferred;
5887
5888                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5889             }
5890
5891         };
5892
5893         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5894         my $err = $@;
5895         die "copy failed: $err" if $err;
5896     }
5897 }
5898
5899 sub qemu_img_format {
5900     my ($scfg, $volname) = @_;
5901
5902     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5903         return $1;
5904     } else {
5905         return "raw";
5906     }
5907 }
5908
5909 sub qemu_drive_mirror {
5910     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete) = @_;
5911
5912     $jobs = {} if !$jobs;
5913
5914     my $qemu_target;
5915     my $format;
5916     $jobs->{"drive-$drive"} = {};
5917
5918     if($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5919         my $server = $1;
5920         my $port = $2;
5921         my $exportname = $3;
5922
5923         $format = "nbd";
5924         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5925         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5926         my $cmd = ['socat', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=1"];
5927
5928         my $pid = fork();
5929         if (!defined($pid)) {
5930             die "forking socat tunnel failed";
5931         } elsif ($pid == 0) {
5932             exec(@$cmd);
5933             exit(-1);
5934         } else {
5935
5936             $jobs->{"drive-$drive"}->{pid} = $pid;
5937
5938             my $timeout = 0;
5939             while (1) {
5940                 last if -S $unixsocket;
5941                 die if $timeout > 5;
5942                 $timeout++;
5943                 sleep 1;
5944             }
5945         }
5946     } else {
5947
5948         my $storecfg = PVE::Storage::config();
5949         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5950
5951         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5952
5953         $format = qemu_img_format($dst_scfg, $dst_volname);
5954
5955         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5956
5957         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5958     }
5959
5960     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5961     $opts->{format} = $format if $format;
5962
5963     print "drive mirror is starting for drive-$drive\n";
5964
5965     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5966
5967     if (my $err = $@) {
5968         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5969         die "mirroring error: $err";
5970     }
5971
5972
5973     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete);
5974 }
5975
5976 sub qemu_drive_mirror_monitor {
5977     my ($vmid, $vmiddst, $jobs, $skipcomplete) = @_;
5978
5979     eval {
5980
5981         my $err_complete = 0;
5982
5983         while (1) {
5984             die "storage migration timed out\n" if $err_complete > 300;
5985
5986             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5987
5988             my $running_mirror_jobs = {};
5989             foreach my $stat (@$stats) {
5990                 next if $stat->{type} ne 'mirror';
5991                 $running_mirror_jobs->{$stat->{device}} = $stat;
5992             }
5993
5994             my $readycounter = 0;
5995
5996             foreach my $job (keys %$jobs) {
5997
5998                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
5999                     print "$job : finished\n";
6000                     delete $jobs->{$job};
6001                     next;
6002                 }
6003
6004                 die "$job: mirroring has been cancelled. Maybe do you have bad sectors?" if !defined($running_mirror_jobs->{$job});
6005
6006                 my $busy = $running_mirror_jobs->{$job}->{busy};
6007                 my $ready = $running_mirror_jobs->{$job}->{ready};
6008                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6009                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6010                     my $remaining = $total - $transferred;
6011                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6012
6013                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6014                 }
6015
6016                 $readycounter++ if $running_mirror_jobs->{$job}->{ready} eq 'true';
6017             }
6018
6019             last if scalar(keys %$jobs) == 0;
6020
6021             if ($readycounter == scalar(keys %$jobs)) {
6022                 print "all mirroring jobs are ready \n";
6023                 last if $skipcomplete; #do the complete later
6024
6025                 if ($vmiddst && $vmiddst != $vmid) {
6026                     # if we clone a disk for a new target vm, we don't switch the disk
6027                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6028                     last;
6029                 } else {
6030
6031                     foreach my $job (keys %$jobs) {
6032                         # try to switch the disk if source and destination are on the same guest
6033                         print "$job : Try to complete block job\n";
6034
6035                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6036                         if ($@ =~ m/cannot be completed/) {
6037                             print "$job : block job cannot be complete. Try again \n";
6038                             $err_complete++;
6039                         }else {
6040                             print "$job : complete ok : flushing pending writes\n";
6041                             $jobs->{$job}->{complete} = 1;
6042                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6043                         }
6044                     }
6045                 }
6046             }
6047             sleep 1;
6048         }
6049     };
6050     my $err = $@;
6051
6052     if ($err) {
6053         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6054         die "mirroring error: $err";
6055     }
6056
6057 }
6058
6059 sub qemu_blockjobs_cancel {
6060     my ($vmid, $jobs) = @_;
6061
6062     foreach my $job (keys %$jobs) {
6063         print "$job: try to cancel block job\n";
6064         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6065         $jobs->{$job}->{cancel} = 1;
6066     }
6067
6068     while (1) {
6069         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6070
6071         my $running_jobs = {};
6072         foreach my $stat (@$stats) {
6073             $running_jobs->{$stat->{device}} = $stat;
6074         }
6075
6076         foreach my $job (keys %$jobs) {
6077
6078             if(defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6079                 print "$job : finished\n";
6080                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6081                 delete $jobs->{$job};
6082             }
6083         }
6084
6085         last if scalar(keys %$jobs) == 0;
6086
6087         sleep 1;
6088     }
6089 }
6090
6091 sub qemu_blockjobs_finish_tunnel {
6092    my ($vmid, $job, $cpid) = @_;
6093
6094    return if !$cpid;
6095
6096    for (my $i = 1; $i < 20; $i++) {
6097         my $waitpid = waitpid($cpid, WNOHANG);
6098         last if (defined($waitpid) && ($waitpid == $cpid));
6099
6100         if ($i == 10) {
6101             kill(15, $cpid);
6102         } elsif ($i >= 15) {
6103             kill(9, $cpid);
6104         }
6105         sleep (1);
6106     }
6107     unlink "/run/qemu-server/$vmid.mirror-$job";
6108 }
6109
6110 sub clone_disk {
6111     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6112         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete) = @_;
6113
6114     my $newvolid;
6115
6116     if (!$full) {
6117         print "create linked clone of drive $drivename ($drive->{file})\n";
6118         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6119         push @$newvollist, $newvolid;
6120     } else {
6121
6122         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6123         $storeid = $storage if $storage;
6124
6125         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6126         if (!$format) {
6127             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6128             $format = qemu_img_format($scfg, $volname);
6129         }
6130
6131         # test if requested format is supported - else use default
6132         my $supported = grep { $_ eq $format } @$validFormats;
6133         $format = $defFormat if !$supported;
6134
6135         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6136
6137         print "create full clone of drive $drivename ($drive->{file})\n";
6138         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6139         push @$newvollist, $newvolid;
6140
6141         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6142
6143         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6144         if (!$running || $snapname) {
6145             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6146         } else {
6147
6148             my $kvmver = get_running_qemu_version ($vmid);
6149             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6150                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6151                     if $drive->{iothread};
6152             }
6153
6154             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete);
6155         }
6156     }
6157
6158     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6159
6160     my $disk = $drive;
6161     $disk->{format} = undef;
6162     $disk->{file} = $newvolid;
6163     $disk->{size} = $size;
6164
6165     return $disk;
6166 }
6167
6168 # this only works if VM is running
6169 sub get_current_qemu_machine {
6170     my ($vmid) = @_;
6171
6172     my $cmd = { execute => 'query-machines', arguments => {} };
6173     my $res = vm_qmp_command($vmid, $cmd);
6174
6175     my ($current, $default);
6176     foreach my $e (@$res) {
6177         $default = $e->{name} if $e->{'is-default'};
6178         $current = $e->{name} if $e->{'is-current'};
6179     }
6180
6181     # fallback to the default machine if current is not supported by qemu
6182     return $current || $default || 'pc';
6183 }
6184
6185 sub get_running_qemu_version {
6186     my ($vmid) = @_;
6187     my $cmd = { execute => 'query-version', arguments => {} };
6188     my $res = vm_qmp_command($vmid, $cmd);
6189     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6190 }
6191
6192 sub qemu_machine_feature_enabled {
6193     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6194
6195     my $current_major;
6196     my $current_minor;
6197
6198     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6199
6200         $current_major = $3;
6201         $current_minor = $4;
6202
6203     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6204
6205         $current_major = $1;
6206         $current_minor = $2;
6207     }
6208
6209     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6210
6211
6212 }
6213
6214 sub qemu_machine_pxe {
6215     my ($vmid, $conf, $machine) = @_;
6216
6217     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6218
6219     foreach my $opt (keys %$conf) {
6220         next if $opt !~ m/^net(\d+)$/;
6221         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6222         next if !$net;
6223         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6224         return $machine.".pxe" if $romfile =~ m/pxe/;
6225         last;
6226     }
6227
6228     return $machine;
6229 }
6230
6231 sub qemu_use_old_bios_files {
6232     my ($machine_type) = @_;
6233
6234     return if !$machine_type;
6235
6236     my $use_old_bios_files = undef;
6237
6238     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6239         $machine_type = $1;
6240         $use_old_bios_files = 1;
6241     } else {
6242         my $kvmver = kvm_user_version();
6243         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6244         # load new efi bios files on migration. So this hack is required to allow
6245         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6246         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6247         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6248     }
6249
6250     return ($use_old_bios_files, $machine_type);
6251 }
6252
6253 sub lspci {
6254
6255     my $devices = {};
6256
6257     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6258             my (undef, $id, $function) = @_;
6259             my $res = { id => $id, function => $function};
6260             push @{$devices->{$id}}, $res;
6261     });
6262
6263     return $devices;
6264 }
6265
6266 sub vm_iothreads_list {
6267     my ($vmid) = @_;
6268
6269     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6270
6271     my $iothreads = {};
6272     foreach my $iothread (@$res) {
6273         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6274     }
6275
6276     return $iothreads;
6277 }
6278
6279 sub scsihw_infos {
6280     my ($conf, $drive) = @_;
6281
6282     my $maxdev = 0;
6283
6284     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6285         $maxdev = 7;
6286     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6287         $maxdev = 1;
6288     } else {
6289         $maxdev = 256;
6290     }
6291
6292     my $controller = int($drive->{index} / $maxdev);
6293     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6294
6295     return ($maxdev, $controller, $controller_prefix);
6296 }
6297
6298 sub add_hyperv_enlighments {
6299     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6300
6301     return if $nokvm;
6302     return if $winversion < 6;
6303     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6304
6305     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6306
6307     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6308         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6309         push @$cpuFlags , 'hv_vapic';
6310         push @$cpuFlags , 'hv_time';
6311     } else {
6312         push @$cpuFlags , 'hv_spinlocks=0xffff';
6313     }
6314
6315     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6316         push @$cpuFlags , 'hv_reset';
6317         push @$cpuFlags , 'hv_vpindex';
6318         push @$cpuFlags , 'hv_runtime';
6319     }
6320
6321     if ($winversion >= 7) {
6322         push @$cpuFlags , 'hv_relaxed';
6323     }
6324 }
6325
6326 sub windows_version {
6327     my ($ostype) = @_;
6328
6329     return 0 if !$ostype;
6330
6331     my $winversion = 0;
6332
6333     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6334         $winversion = 5;
6335     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6336         $winversion = 6;
6337     } elsif ($ostype =~ m/^win(\d+)$/) {
6338         $winversion = $1;
6339     }
6340
6341     return $winversion;
6342 }
6343
6344 # bash completion helper
6345
6346 sub complete_backup_archives {
6347     my ($cmdname, $pname, $cvalue) = @_;
6348
6349     my $cfg = PVE::Storage::config();
6350
6351     my $storeid;
6352
6353     if ($cvalue =~ m/^([^:]+):/) {
6354         $storeid = $1;
6355     }
6356
6357     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6358
6359     my $res = [];
6360     foreach my $id (keys %$data) {
6361         foreach my $item (@{$data->{$id}}) {
6362             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6363             push @$res, $item->{volid} if defined($item->{volid});
6364         }
6365     }
6366
6367     return $res;
6368 }
6369
6370 my $complete_vmid_full = sub {
6371     my ($running) = @_;
6372
6373     my $idlist = vmstatus();
6374
6375     my $res = [];
6376
6377     foreach my $id (keys %$idlist) {
6378         my $d = $idlist->{$id};
6379         if (defined($running)) {
6380             next if $d->{template};
6381             next if $running && $d->{status} ne 'running';
6382             next if !$running && $d->{status} eq 'running';
6383         }
6384         push @$res, $id;
6385
6386     }
6387     return $res;
6388 };
6389
6390 sub complete_vmid {
6391     return &$complete_vmid_full();
6392 }
6393
6394 sub complete_vmid_stopped {
6395     return &$complete_vmid_full(0);
6396 }
6397
6398 sub complete_vmid_running {
6399     return &$complete_vmid_full(1);
6400 }
6401
6402 sub complete_storage {
6403
6404     my $cfg = PVE::Storage::config();
6405     my $ids = $cfg->{ids};
6406
6407     my $res = [];
6408     foreach my $sid (keys %$ids) {
6409         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6410         next if !$ids->{$sid}->{content}->{images};
6411         push @$res, $sid;
6412     }
6413
6414     return $res;
6415 }
6416
6417 sub nbd_stop {
6418     my ($vmid) = @_;
6419
6420     vm_mon_cmd($vmid, 'nbd-server-stop');
6421 }
6422
6423 1;