PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         default => 'kvm64',
 151         default_key => 1,
 152     },
 153     hidden => {
 154         description => "Do not identify as a KVM virtual machine.",
 155         type => 'boolean',
 156         optional => 1,
 157         default => 0
 158     },
 159 };
 160
 161 my $watchdog_fmt = {
 162     model => {
 163         default_key => 1,
 164         type => 'string',
 165         enum => [qw(i6300esb ib700)],
 166         description => "Watchdog type to emulate.",
 167         default => 'i6300esb',
 168         optional => 1,
 169     },
 170     action => {
 171         type => 'string',
 172         enum => [qw(reset shutdown poweroff pause debug none)],
 173         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 174         optional => 1,
 175     },
 176 };
 177 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 178
 179 my $confdesc = {
 180     onboot => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Specifies whether a VM will be started during system bootup.",
 184         default => 0,
 185     },
 186     autostart => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Automatic restart after crash (currently ignored).",
 190         default => 0,
 191     },
 192     hotplug => {
 193         optional => 1,
 194         type => 'string', format => 'pve-hotplug-features',
 195         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 196         default => 'network,disk,usb',
 197     },
 198     reboot => {
 199         optional => 1,
 200         type => 'boolean',
 201         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 202         default => 1,
 203     },
 204     lock => {
 205         optional => 1,
 206         type => 'string',
 207         description => "Lock/unlock the VM.",
 208         enum => [qw(migrate backup snapshot rollback)],
 209     },
 210     cpulimit => {
 211         optional => 1,
 212         type => 'number',
 213         description => "Limit of CPU usage.",
 214         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 215         minimum => 0,
 216         maximum => 128,
 217         default => 0,
 218     },
 219     cpuunits => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "CPU weight for a VM.",
 223         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 224         minimum => 0,
 225         maximum => 500000,
 226         default => 1024,
 227     },
 228     memory => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 232         minimum => 16,
 233         default => 512,
 234     },
 235     balloon => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 239         minimum => 0,
 240     },
 241     shares => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 245         minimum => 0,
 246         maximum => 50000,
 247         default => 1000,
 248     },
 249     keyboard => {
 250         optional => 1,
 251         type => 'string',
 252         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 253         enum => PVE::Tools::kvmkeymaplist(),
 254         default => 'en-us',
 255     },
 256     name => {
 257         optional => 1,
 258         type => 'string', format => 'dns-name',
 259         description => "Set a name for the VM. Only used on the configuration web interface.",
 260     },
 261     scsihw => {
 262         optional => 1,
 263         type => 'string',
 264         description => "SCSI controller model",
 265         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 266         default => 'lsi',
 267     },
 268     description => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 272     },
 273     ostype => {
 274         optional => 1,
 275         type => 'string',
 276         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 277         description => "Specify guest operating system.",
 278         verbose_description => <<EODESC,
 279 Specify guest operating system. This is used to enable special
 280 optimization/features for specific operating systems:
 281
 282 [horizontal]
 283 other;; unspecified OS
 284 wxp;; Microsoft Windows XP
 285 w2k;; Microsoft Windows 2000
 286 w2k3;; Microsoft Windows 2003
 287 w2k8;; Microsoft Windows 2008
 288 wvista;; Microsoft Windows Vista
 289 win7;; Microsoft Windows 7
 290 win8;; Microsoft Windows 8/2012
 291 l24;; Linux 2.4 Kernel
 292 l26;; Linux 2.6/3.X Kernel
 293 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 294 EODESC
 295     },
 296     boot => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 300         pattern => '[acdn]{1,4}',
 301         default => 'cdn',
 302     },
 303     bootdisk => {
 304         optional => 1,
 305         type => 'string', format => 'pve-qm-bootdisk',
 306         description => "Enable booting from specified disk.",
 307         pattern => '(ide|sata|scsi|virtio)\d+',
 308     },
 309     smp => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of CPUs. Please use option -sockets instead.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     sockets => {
 317         optional => 1,
 318         type => 'integer',
 319         description => "The number of CPU sockets.",
 320         minimum => 1,
 321         default => 1,
 322     },
 323     cores => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "The number of cores per socket.",
 327         minimum => 1,
 328         default => 1,
 329     },
 330     numa => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable NUMA.",
 334         default => 0,
 335     },
 336     hugepages => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Enable/disable hugepages memory.",
 340         enum => [qw(any 2 1024)],
 341     },
 342     vcpus => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Number of hotplugged vcpus.",
 346         minimum => 1,
 347         default => 0,
 348     },
 349     acpi => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable ACPI.",
 353         default => 1,
 354     },
 355     agent => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable Qemu GuestAgent.",
 359         default => 0,
 360     },
 361     kvm => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable KVM hardware virtualization.",
 365         default => 1,
 366     },
 367     tdf => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable time drift fix.",
 371         default => 0,
 372     },
 373     localtime => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 377     },
 378     freeze => {
 379         optional => 1,
 380         type => 'boolean',
 381         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 382     },
 383     vga => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Select the VGA type.",
 387         verbose_description => "Select the VGA type. If you want to use high resolution" .
 388             " modes (>= 1280x1024x16) then you should use the options " .
 389             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 390             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 391             "display sever. For win* OS you can select how many independent " .
 392             "displays you want, Linux guests can add displays them self. " .
 393             "You can also run without any graphic card, using a serial device" .
 394             " as terminal.",
 395         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 396     },
 397     watchdog => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-watchdog',
 400         description => "Create a virtual hardware watchdog device.",
 401         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 402             " (by a guest action), the watchdog must be periodically polled " .
 403             "by an agent inside the guest or else the watchdog will reset " .
 404             "the guest (or execute the respective action specified)",
 405     },
 406     startdate => {
 407         optional => 1,
 408         type => 'string',
 409         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 410         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 411         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 412         default => 'now',
 413     },
 414     startup =>  get_standard_option('pve-startup-order'),
 415     template => {
 416         optional => 1,
 417         type => 'boolean',
 418         description => "Enable/disable Template.",
 419         default => 0,
 420     },
 421     args => {
 422         optional => 1,
 423         type => 'string',
 424         description => "Arbitrary arguments passed to kvm.",
 425         verbose_description => <<EODESCR,
 426 Arbitrary arguments passed to kvm, for example:
 427
 428 args: -no-reboot -no-hpet
 429
 430 NOTE: this option is for experts only.
 431 EODESCR
 432     },
 433     tablet => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 1,
 437         description => "Enable/disable the USB tablet device.",
 438         verbose_description => "Enable/disable the USB tablet device. This device is " .
 439             "usually needed to allow absolute mouse positioning with VNC. " .
 440             "Else the mouse runs out of sync with normal VNC clients. " .
 441             "If you're running lots of console-only guests on one host, " .
 442             "you may consider disabling this to save some context switches. " .
 443             "This is turned off by default if you use spice (-vga=qxl).",
 444     },
 445     migrate_speed => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 449         minimum => 0,
 450         default => 0,
 451     },
 452     migrate_downtime => {
 453         optional => 1,
 454         type => 'number',
 455         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 456         minimum => 0,
 457         default => 0.1,
 458     },
 459     cdrom => {
 460         optional => 1,
 461         type => 'string', format => 'pve-qm-ide',
 462         typetext => '<volume>',
 463         description => "This is an alias for option -ide2",
 464     },
 465     cpu => {
 466         optional => 1,
 467         description => "Emulated CPU type.",
 468         type => 'string',
 469         format => $cpu_fmt,
 470     },
 471     parent => get_standard_option('pve-snapshot-name', {
 472         optional => 1,
 473         description => "Parent snapshot name. This is used internally, and should not be modified.",
 474     }),
 475     snaptime => {
 476         optional => 1,
 477         description => "Timestamp for snapshots.",
 478         type => 'integer',
 479         minimum => 0,
 480     },
 481     vmstate => {
 482         optional => 1,
 483         type => 'string', format => 'pve-volume-id',
 484         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 485     },
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     werror => {
 756         type => 'string',
 757         enum => [qw(enospc ignore report stop)],
 758         description => 'Write error action.',
 759         optional => 1,
 760     },
 761     aio => {
 762         type => 'string',
 763         enum => [qw(native threads)],
 764         description => 'AIO type to use.',
 765         optional => 1,
 766     },
 767     discard => {
 768         type => 'string',
 769         enum => [qw(ignore on)],
 770         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 771         optional => 1,
 772     },
 773     detect_zeroes => {
 774         type => 'boolean',
 775         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 776         optional => 1,
 777     },
 778     serial => {
 779         type => 'string',
 780         format => 'urlencoded',
 781         format_description => 'serial',
 782         maxLength => 20*3, # *3 since it's %xx url enoded
 783         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 784         optional => 1,
 785     }
 786 );
 787
 788 my %rerror_fmt = (
 789     rerror => {
 790         type => 'string',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         description => "Whether to use iothreads for this drive",
 800         optional => 1,
 801 });
 802
 803 my %model_fmt = (
 804     model => {
 805         type => 'string',
 806         format => 'urlencoded',
 807         format_description => 'model',
 808         maxLength => 40*3, # *3 since it's %xx url enoded
 809         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 810         optional => 1,
 811     },
 812 );
 813
 814 my %queues_fmt = (
 815     queues => {
 816         type => 'integer',
 817         description => "Number of queues.",
 818         minimum => 2,
 819         optional => 1
 820     }
 821 );
 822
 823 my $add_throttle_desc = sub {
 824     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 825     my $d = {
 826         type => $type,
 827         format_description => $unit,
 828         description => "Maximum $what in $longunit.",
 829         optional => 1,
 830     };
 831     $d->{minimum} = $minimum if defined($minimum);
 832     $drivedesc_base{$key} = $d;
 833 };
 834 # throughput: (leaky bucket)
 835 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 836 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 838 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 839 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 841 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 842 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 843 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 844
 845 # pools: (pool of IO before throttling starts taking effect)
 846 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 852
 853 # burst lengths
 854 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 855 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 857 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 860
 861 my $ide_fmt = {
 862     %drivedesc_base,
 863     %rerror_fmt,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879 };
 880 my $scsidesc = {
 881     optional => 1,
 882     type => 'string', format => $scsi_fmt,
 883     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 886
 887 my $sata_fmt = {
 888     %drivedesc_base,
 889     %rerror_fmt,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901     %rerror_fmt,
 902 };
 903 my $virtiodesc = {
 904     optional => 1,
 905     type => 'string', format => $virtio_fmt,
 906     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 909
 910 my $alldrive_fmt = {
 911     %drivedesc_base,
 912     %rerror_fmt,
 913     %iothread_fmt,
 914     %model_fmt,
 915     %queues_fmt,
 916 };
 917
 918 my $efidisk_fmt = {
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     format => get_standard_option('pve-qm-image-format'),
 928     size => {
 929         type => 'string',
 930         format => 'disk-size',
 931         format_description => 'DiskSize',
 932         description => "Disk size. This is purely informational and has no effect.",
 933         optional => 1,
 934     },
 935 };
 936
 937 my $efidisk_desc = {
 938     optional => 1,
 939     type => 'string', format => $efidisk_fmt,
 940     description => "Configure a Disk for storing EFI vars",
 941 };
 942
 943 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 944
 945 my $usb_fmt = {
 946     host => {
 947         default_key => 1,
 948         type => 'string', format => 'pve-qm-usb-device',
 949         format_description => 'HOSTUSBDEVICE|spice',
 950         description => <<EODESCR,
 951 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 952
 953  'bus-port(.port)*' (decimal numbers) or
 954  'vendor_id:product_id' (hexadeciaml numbers) or
 955  'spice'
 956
 957 You can use the 'lsusb -t' command to list existing usb devices.
 958
 959 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 960
 961 The value 'spice' can be used to add a usb redirection devices for spice.
 962 EODESCR
 963     },
 964     usb3 => {
 965         optional => 1,
 966         type => 'boolean',
 967         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 968         default => 0,
 969     },
 970 };
 971
 972 my $usbdesc = {
 973     optional => 1,
 974     type => 'string', format => $usb_fmt,
 975     description => "Configure an USB device (n is 0 to 4).",
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 978
 979 # NOTE: the match-groups of this regex are used in parse_hostpci
 980 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 981 my $hostpci_fmt = {
 982     host => {
 983         default_key => 1,
 984         type => 'string',
 985         pattern => qr/$PCIRE(;$PCIRE)*/,
 986         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 987         description => <<EODESCR,
 988 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 989 of PCI virtual functions of the host. HOSTPCIID syntax is:
 990
 991 'bus:dev.func' (hexadecimal numbers)
 992
 993 You can us the 'lspci' command to list existing PCI devices.
 994 EODESCR
 995     },
 996     rombar => {
 997         type => 'boolean',
 998         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 999         optional => 1,
1000         default => 1,
1001     },
1002     pcie => {
1003         type => 'boolean',
1004         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1005         optional => 1,
1006         default => 0,
1007     },
1008     'x-vga' => {
1009         type => 'boolean',
1010         description =>  "Enable vfio-vga device support.",
1011         optional => 1,
1012         default => 0,
1013     },
1014 };
1015 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1016
1017 my $hostpcidesc = {
1018         optional => 1,
1019         type => 'string', format => 'pve-qm-hostpci',
1020         description => "Map host PCI devices into guest.",
1021         verbose_description =>  <<EODESCR,
1022 Map host PCI devices into guest.
1023
1024 NOTE: This option allows direct access to host hardware. So it is no longer
1025 possible to migrate such machines - use with special care.
1026
1027 CAUTION: Experimental! User reported problems with this option.
1028 EODESCR
1029 };
1030 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1031
1032 my $serialdesc = {
1033         optional => 1,
1034         type => 'string',
1035         pattern => '(/dev/.+|socket)',
1036         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1037         verbose_description =>  <<EODESCR,
1038 Create a serial device inside the VM (n is 0 to 3), and pass through a
1039 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1040 host side (use 'qm terminal' to open a terminal connection).
1041
1042 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1043
1044 CAUTION: Experimental! User reported problems with this option.
1045 EODESCR
1046 };
1047
1048 my $paralleldesc= {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1052         description =>  "Map host parallel devices (n is 0 to 2).",
1053         verbose_description =>  <<EODESCR,
1054 Map host parallel devices (n is 0 to 2).
1055
1056 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1057
1058 CAUTION: Experimental! User reported problems with this option.
1059 EODESCR
1060 };
1061
1062 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1063     $confdesc->{"parallel$i"} = $paralleldesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1067     $confdesc->{"serial$i"} = $serialdesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1071     $confdesc->{"hostpci$i"} = $hostpcidesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1075     $drivename_hash->{"ide$i"} = 1;
1076     $confdesc->{"ide$i"} = $idedesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1080     $drivename_hash->{"sata$i"} = 1;
1081     $confdesc->{"sata$i"} = $satadesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1085     $drivename_hash->{"scsi$i"} = 1;
1086     $confdesc->{"scsi$i"} = $scsidesc ;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1090     $drivename_hash->{"virtio$i"} = 1;
1091     $confdesc->{"virtio$i"} = $virtiodesc;
1092 }
1093
1094 $drivename_hash->{efidisk0} = 1;
1095 $confdesc->{efidisk0} = $efidisk_desc;
1096
1097 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1098     $confdesc->{"usb$i"} = $usbdesc;
1099 }
1100
1101 my $unuseddesc = {
1102     optional => 1,
1103     type => 'string', format => 'pve-volume-id',
1104     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1105 };
1106
1107 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1108     $confdesc->{"unused$i"} = $unuseddesc;
1109 }
1110
1111 my $kvm_api_version = 0;
1112
1113 sub kvm_version {
1114
1115     return $kvm_api_version if $kvm_api_version;
1116
1117     my $fh = IO::File->new("</dev/kvm") ||
1118         return 0;
1119
1120     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1121         $kvm_api_version = $v;
1122     }
1123
1124     $fh->close();
1125
1126     return  $kvm_api_version;
1127 }
1128
1129 my $kvm_user_version;
1130
1131 sub kvm_user_version {
1132
1133     return $kvm_user_version if $kvm_user_version;
1134
1135     $kvm_user_version = 'unknown';
1136
1137     my $code = sub {
1138         my $line = shift;
1139         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1140             $kvm_user_version = $2;
1141         }
1142     };
1143
1144     eval { run_command("kvm -version", outfunc => $code); };
1145     warn $@ if $@;
1146
1147     return $kvm_user_version;
1148
1149 }
1150
1151 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1152
1153 sub valid_drive_names {
1154     # order is important - used to autoselect boot disk
1155     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1156             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1157             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1158             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1159             'efidisk0');
1160 }
1161
1162 sub is_valid_drivename {
1163     my $dev = shift;
1164
1165     return defined($drivename_hash->{$dev});
1166 }
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 sub nic_models {
1174     return $nic_model_list;
1175 }
1176
1177 sub os_list_description {
1178
1179     return {
1180         other => 'Other',
1181         wxp => 'Windows XP',
1182         w2k => 'Windows 2000',
1183         w2k3 =>, 'Windows 2003',
1184         w2k8 => 'Windows 2008',
1185         wvista => 'Windows Vista',
1186         win7 => 'Windows 7',
1187         win8 => 'Windows 8/2012',
1188         win10 => 'Windows 10/2016',
1189         l24 => 'Linux 2.4',
1190         l26 => 'Linux 2.6',
1191     };
1192 }
1193
1194 my $cdrom_path;
1195
1196 sub get_cdrom_path {
1197
1198     return  $cdrom_path if $cdrom_path;
1199
1200     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1201     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1202     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1203 }
1204
1205 sub get_iso_path {
1206     my ($storecfg, $vmid, $cdrom) = @_;
1207
1208     if ($cdrom eq 'cdrom') {
1209         return get_cdrom_path();
1210     } elsif ($cdrom eq 'none') {
1211         return '';
1212     } elsif ($cdrom =~ m|^/|) {
1213         return $cdrom;
1214     } else {
1215         return PVE::Storage::path($storecfg, $cdrom);
1216     }
1217 }
1218
1219 # try to convert old style file names to volume IDs
1220 sub filename_to_volume_id {
1221     my ($vmid, $file, $media) = @_;
1222
1223     if (!($file eq 'none' || $file eq 'cdrom' ||
1224           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1225
1226         return undef if $file =~ m|/|;
1227
1228         if ($media && $media eq 'cdrom') {
1229             $file = "local:iso/$file";
1230         } else {
1231             $file = "local:$vmid/$file";
1232         }
1233     }
1234
1235     return $file;
1236 }
1237
1238 sub verify_media_type {
1239     my ($opt, $vtype, $media) = @_;
1240
1241     return if !$media;
1242
1243     my $etype;
1244     if ($media eq 'disk') {
1245         $etype = 'images';
1246     } elsif ($media eq 'cdrom') {
1247         $etype = 'iso';
1248     } else {
1249         die "internal error";
1250     }
1251
1252     return if ($vtype eq $etype);
1253
1254     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1255 }
1256
1257 sub cleanup_drive_path {
1258     my ($opt, $storecfg, $drive) = @_;
1259
1260     # try to convert filesystem paths to volume IDs
1261
1262     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1263         ($drive->{file} !~ m|^/dev/.+|) &&
1264         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1265         ($drive->{file} !~ m/^\d+$/)) {
1266         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1267         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1268         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1269         verify_media_type($opt, $vtype, $drive->{media});
1270         $drive->{file} = $volid;
1271     }
1272
1273     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1274 }
1275
1276 sub parse_hotplug_features {
1277     my ($data) = @_;
1278
1279     my $res = {};
1280
1281     return $res if $data eq '0';
1282
1283     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1284
1285     foreach my $feature (PVE::Tools::split_list($data)) {
1286         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1287             $res->{$1} = 1;
1288         } else {
1289             die "invalid hotplug feature '$feature'\n";
1290         }
1291     }
1292     return $res;
1293 }
1294
1295 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1296 sub pve_verify_hotplug_features {
1297     my ($value, $noerr) = @_;
1298
1299     return $value if parse_hotplug_features($value);
1300
1301     return undef if $noerr;
1302
1303     die "unable to parse hotplug option\n";
1304 }
1305
1306 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1307 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1308 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1309 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1310 #        [,iothread=on][,serial=serial][,model=model]
1311
1312 sub parse_drive {
1313     my ($key, $data) = @_;
1314
1315     my ($interface, $index);
1316
1317     if ($key =~ m/^([^\d]+)(\d+)$/) {
1318         $interface = $1;
1319         $index = $2;
1320     } else {
1321         return undef;
1322     }
1323
1324     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1325                                      : $confdesc->{$key}->{format};
1326     if (!$desc) {
1327         warn "invalid drive key: $key\n";
1328         return undef;
1329     }
1330     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1331     return undef if !$res;
1332     $res->{interface} = $interface;
1333     $res->{index} = $index;
1334
1335     my $error = 0;
1336     foreach my $opt (qw(bps bps_rd bps_wr)) {
1337         if (my $bps = defined(delete $res->{$opt})) {
1338             if (defined($res->{"m$opt"})) {
1339                 warn "both $opt and m$opt specified\n";
1340                 ++$error;
1341                 next;
1342             }
1343             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1344         }
1345     }
1346
1347     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1348     for my $requirement (
1349         [bps_max_length => 'mbps_max'],
1350         [bps_rd_max_length => 'mbps_rd_max'],
1351         [bps_wr_max_length => 'mbps_wr_max'],
1352         [iops_max_length => 'iops_max'],
1353         [iops_rd_max_length => 'iops_rd_max'],
1354         [iops_wr_max_length => 'iops_wr_max']) {
1355         my ($option, $requires) = @$requirement;
1356         if ($res->{$option} && !$res->{$requires}) {
1357             warn "$option requires $requires\n";
1358             ++$error;
1359         }
1360     }
1361
1362     return undef if $error;
1363
1364     return undef if $res->{mbps_rd} && $res->{mbps};
1365     return undef if $res->{mbps_wr} && $res->{mbps};
1366     return undef if $res->{iops_rd} && $res->{iops};
1367     return undef if $res->{iops_wr} && $res->{iops};
1368
1369     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1370         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1371         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1372         return undef if $res->{interface} eq 'virtio';
1373     }
1374
1375     if (my $size = $res->{size}) {
1376         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1377     }
1378
1379     return $res;
1380 }
1381
1382 sub print_drive {
1383     my ($vmid, $drive) = @_;
1384     my $data = { %$drive };
1385     delete $data->{$_} for qw(index interface);
1386     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1387 }
1388
1389 sub scsi_inquiry {
1390     my($fh, $noerr) = @_;
1391
1392     my $SG_IO = 0x2285;
1393     my $SG_GET_VERSION_NUM = 0x2282;
1394
1395     my $versionbuf = "\x00" x 8;
1396     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1397     if (!$ret) {
1398         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1399         return undef;
1400     }
1401     my $version = unpack("I", $versionbuf);
1402     if ($version < 30000) {
1403         die "scsi generic interface too old\n"  if !$noerr;
1404         return undef;
1405     }
1406
1407     my $buf = "\x00" x 36;
1408     my $sensebuf = "\x00" x 8;
1409     my $cmd = pack("C x3 C x1", 0x12, 36);
1410
1411     # see /usr/include/scsi/sg.h
1412     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1413
1414     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1415                       length($sensebuf), 0, length($buf), $buf,
1416                       $cmd, $sensebuf, 6000);
1417
1418     $ret = ioctl($fh, $SG_IO, $packet);
1419     if (!$ret) {
1420         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1421         return undef;
1422     }
1423
1424     my @res = unpack($sg_io_hdr_t, $packet);
1425     if ($res[17] || $res[18]) {
1426         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my $res = {};
1431     (my $byte0, my $byte1, $res->{vendor},
1432      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1433
1434     $res->{removable} = $byte1 & 128 ? 1 : 0;
1435     $res->{type} = $byte0 & 31;
1436
1437     return $res;
1438 }
1439
1440 sub path_is_scsi {
1441     my ($path) = @_;
1442
1443     my $fh = IO::File->new("+<$path") || return undef;
1444     my $res = scsi_inquiry($fh, 1);
1445     close($fh);
1446
1447     return $res;
1448 }
1449
1450 sub machine_type_is_q35 {
1451     my ($conf) = @_;
1452
1453     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1454 }
1455
1456 sub print_tabletdevice_full {
1457     my ($conf) = @_;
1458
1459     my $q35 = machine_type_is_q35($conf);
1460
1461     # we use uhci for old VMs because tablet driver was buggy in older qemu
1462     my $usbbus = $q35 ? "ehci" : "uhci";
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_drivedevice_full {
1468     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1469
1470     my $device = '';
1471     my $maxdev = 0;
1472
1473     if ($drive->{interface} eq 'virtio') {
1474         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1475         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1476         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1477     } elsif ($drive->{interface} eq 'scsi') {
1478
1479         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1480         my $unit = $drive->{index} % $maxdev;
1481         my $devicetype = 'hd';
1482         my $path = '';
1483         if (drive_is_cdrom($drive)) {
1484             $devicetype = 'cd';
1485         } else {
1486             if ($drive->{file} =~ m|^/|) {
1487                 $path = $drive->{file};
1488                 if (my $info = path_is_scsi($path)) {
1489                     if ($info->{type} == 0) {
1490                         $devicetype = 'block';
1491                     } elsif ($info->{type} == 1) { # tape
1492                         $devicetype = 'generic';
1493                     }
1494                 }
1495             } else {
1496                  $path = PVE::Storage::path($storecfg, $drive->{file});
1497             }
1498
1499             if($path =~ m/^iscsi\:\/\//){
1500                 $devicetype = 'generic';
1501             }
1502         }
1503
1504         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1505             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1506         } else {
1507             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1508         }
1509
1510     } elsif ($drive->{interface} eq 'ide'){
1511         $maxdev = 2;
1512         my $controller = int($drive->{index} / $maxdev);
1513         my $unit = $drive->{index} % $maxdev;
1514         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1515
1516         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1517         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1518             $model = URI::Escape::uri_unescape($model);
1519             $device .= ",model=$model";
1520         }
1521     } elsif ($drive->{interface} eq 'sata'){
1522         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1523         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1524         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1525     } elsif ($drive->{interface} eq 'usb') {
1526         die "implement me";
1527         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1528     } else {
1529         die "unsupported interface type";
1530     }
1531
1532     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1533
1534     return $device;
1535 }
1536
1537 sub get_initiator_name {
1538     my $initiator;
1539
1540     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1541     while (defined(my $line = <$fh>)) {
1542         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1543         $initiator = $1;
1544         last;
1545     }
1546     $fh->close();
1547
1548     return $initiator;
1549 }
1550
1551 sub print_drive_full {
1552     my ($storecfg, $vmid, $drive) = @_;
1553
1554     my $path;
1555     my $volid = $drive->{file};
1556     my $format;
1557
1558     if (drive_is_cdrom($drive)) {
1559         $path = get_iso_path($storecfg, $vmid, $volid);
1560     } else {
1561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562         if ($storeid) {
1563             $path = PVE::Storage::path($storecfg, $volid);
1564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1565             $format = qemu_img_format($scfg, $volname);
1566         } else {
1567             $path = $volid;
1568             $format = "raw";
1569         }
1570    }
1571
1572     my $opts = '';
1573     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1574     foreach my $o (@qemu_drive_options) {
1575         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1576     }
1577     if (my $serial = $drive->{serial}) {
1578         $serial = URI::Escape::uri_unescape($serial);
1579         $opts .= ",serial=$serial";
1580     }
1581
1582     $opts .= ",format=$format" if $format && !$drive->{format};
1583
1584     foreach my $o (qw(bps bps_rd bps_wr)) {
1585         my $v = $drive->{"m$o"};
1586         $opts .= ",$o=" . int($v*1024*1024) if $v;
1587     }
1588
1589     my $cache_direct = 0;
1590
1591     if (my $cache = $drive->{cache}) {
1592         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1593     } elsif (!drive_is_cdrom($drive)) {
1594         $opts .= ",cache=none";
1595         $cache_direct = 1;
1596     }
1597
1598     # aio native works only with O_DIRECT
1599     if (!$drive->{aio}) {
1600         if($cache_direct) {
1601             $opts .= ",aio=native";
1602         } else {
1603             $opts .= ",aio=threads";
1604         }
1605     }
1606
1607     if (!drive_is_cdrom($drive)) {
1608         my $detectzeroes;
1609         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1610             $detectzeroes = 'off';
1611         } elsif ($drive->{discard}) {
1612             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1613         } else {
1614             # This used to be our default with discard not being specified:
1615             $detectzeroes = 'on';
1616         }
1617         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1618     }
1619
1620     my $pathinfo = $path ? "file=$path," : '';
1621
1622     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1623 }
1624
1625 sub print_netdevice_full {
1626     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1627
1628     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1629
1630     my $device = $net->{model};
1631     if ($net->{model} eq 'virtio') {
1632          $device = 'virtio-net-pci';
1633      };
1634
1635     my $pciaddr = print_pci_addr("$netid", $bridges);
1636     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1637     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1638         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1639         my $vectors = $net->{queues} * 2 + 2;
1640         $tmpstr .= ",vectors=$vectors,mq=on";
1641     }
1642     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1643
1644     if ($use_old_bios_files) {
1645         my $romfile;
1646         if ($device eq 'virtio-net-pci') {
1647             $romfile = 'pxe-virtio.rom';
1648         } elsif ($device eq 'e1000') {
1649             $romfile = 'pxe-e1000.rom';
1650         } elsif ($device eq 'ne2k') {
1651             $romfile = 'pxe-ne2k_pci.rom';
1652         } elsif ($device eq 'pcnet') {
1653             $romfile = 'pxe-pcnet.rom';
1654         } elsif ($device eq 'rtl8139') {
1655             $romfile = 'pxe-rtl8139.rom';
1656         }
1657         $tmpstr .= ",romfile=$romfile" if $romfile;
1658     }
1659
1660     return $tmpstr;
1661 }
1662
1663 sub print_netdev_full {
1664     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1665
1666     my $i = '';
1667     if ($netid =~ m/^net(\d+)$/) {
1668         $i = int($1);
1669     }
1670
1671     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1672
1673     my $ifname = "tap${vmid}i$i";
1674
1675     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1676     die "interface name '$ifname' is too long (max 15 character)\n"
1677         if length($ifname) >= 16;
1678
1679     my $vhostparam = '';
1680     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1681
1682     my $vmname = $conf->{name} || "vm$vmid";
1683
1684     my $netdev = "";
1685     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1686
1687     if ($net->{bridge}) {
1688         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1689     } else {
1690         $netdev = "type=user,id=$netid,hostname=$vmname";
1691     }
1692
1693     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1694
1695     return $netdev;
1696 }
1697
1698
1699 sub print_cpu_device {
1700     my ($conf, $id) = @_;
1701
1702     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1703     my $cpu = $nokvm ? "qemu64" : "kvm64";
1704     if (my $cputype = $conf->{cpu}) {
1705         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1706             or die "Cannot parse cpu description: $cputype\n";
1707         $cpu = $cpuconf->{cputype};
1708     }
1709
1710     my $sockets = 1;
1711     $sockets = $conf->{sockets} if  $conf->{sockets};
1712     my $cores = $conf->{cores} || 1;
1713
1714     my $current_core = ($id - 1) % $cores;
1715     my $current_socket = int(($id - $current_core)/$cores);
1716
1717     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1718 }
1719
1720 sub drive_is_cdrom {
1721     my ($drive) = @_;
1722
1723     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1724
1725 }
1726
1727 sub parse_number_sets {
1728     my ($set) = @_;
1729     my $res = [];
1730     foreach my $part (split(/;/, $set)) {
1731         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1732             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1733             push @$res, [ $1, $2 ];
1734         } else {
1735             die "invalid range: $part\n";
1736         }
1737     }
1738     return $res;
1739 }
1740
1741 sub parse_numa {
1742     my ($data) = @_;
1743
1744     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1745     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1746     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1747     return $res;
1748 }
1749
1750 sub parse_hostpci {
1751     my ($value) = @_;
1752
1753     return undef if !$value;
1754
1755     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1756
1757     my @idlist = split(/;/, $res->{host});
1758     delete $res->{host};
1759     foreach my $id (@idlist) {
1760         if ($id =~ /^$PCIRE$/) {
1761             if (defined($2)) {
1762                 push @{$res->{pciid}}, { id => $1, function => $2 };
1763             } else {
1764                 my $pcidevices = lspci($1);
1765                 $res->{pciid} = $pcidevices->{$1};
1766             }
1767         } else {
1768             # should have been caught by parse_property_string already
1769             die "failed to parse PCI id: $id\n";
1770         }
1771     }
1772     return $res;
1773 }
1774
1775 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1776 sub parse_net {
1777     my ($data) = @_;
1778
1779     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1780     if ($@) {
1781         warn $@;
1782         return undef;
1783     }
1784     if (!defined($res->{macaddr})) {
1785         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1786         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1787     }
1788     return $res;
1789 }
1790
1791 sub print_net {
1792     my $net = shift;
1793
1794     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1795 }
1796
1797 sub add_random_macs {
1798     my ($settings) = @_;
1799
1800     foreach my $opt (keys %$settings) {
1801         next if $opt !~ m/^net(\d+)$/;
1802         my $net = parse_net($settings->{$opt});
1803         next if !$net;
1804         $settings->{$opt} = print_net($net);
1805     }
1806 }
1807
1808 sub vm_is_volid_owner {
1809     my ($storecfg, $vmid, $volid) = @_;
1810
1811     if ($volid !~  m|^/|) {
1812         my ($path, $owner);
1813         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1814         if ($owner && ($owner == $vmid)) {
1815             return 1;
1816         }
1817     }
1818
1819     return undef;
1820 }
1821
1822 sub split_flagged_list {
1823     my $text = shift || '';
1824     $text =~ s/[,;]/ /g;
1825     $text =~ s/^\s+//;
1826     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1827 }
1828
1829 sub join_flagged_list {
1830     my ($how, $lst) = @_;
1831     join $how, map { $lst->{$_} . $_ } keys %$lst;
1832 }
1833
1834 sub vmconfig_delete_pending_option {
1835     my ($conf, $key, $force) = @_;
1836
1837     delete $conf->{pending}->{$key};
1838     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1839     $pending_delete_hash->{$key} = $force ? '!' : '';
1840     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1841 }
1842
1843 sub vmconfig_undelete_pending_option {
1844     my ($conf, $key) = @_;
1845
1846     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1847     delete $pending_delete_hash->{$key};
1848
1849     if (%$pending_delete_hash) {
1850         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1851     } else {
1852         delete $conf->{pending}->{delete};
1853     }
1854 }
1855
1856 sub vmconfig_register_unused_drive {
1857     my ($storecfg, $vmid, $conf, $drive) = @_;
1858
1859     if (!drive_is_cdrom($drive)) {
1860         my $volid = $drive->{file};
1861         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1862             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1863         }
1864     }
1865 }
1866
1867 sub vmconfig_cleanup_pending {
1868     my ($conf) = @_;
1869
1870     # remove pending changes when nothing changed
1871     my $changes;
1872     foreach my $opt (keys %{$conf->{pending}}) {
1873         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1874             $changes = 1;
1875             delete $conf->{pending}->{$opt};
1876         }
1877     }
1878
1879     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1880     my $pending_delete_hash = {};
1881     while (my ($opt, $force) = each %$current_delete_hash) {
1882         if (defined($conf->{$opt})) {
1883             $pending_delete_hash->{$opt} = $force;
1884         } else {
1885             $changes = 1;
1886         }
1887     }
1888
1889     if (%$pending_delete_hash) {
1890         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1891     } else {
1892         delete $conf->{pending}->{delete};
1893     }
1894
1895     return $changes;
1896 }
1897
1898 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1899 my $smbios1_fmt = {
1900     uuid => {
1901         type => 'string',
1902         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1903         format_description => 'UUID',
1904         description => "Set SMBIOS1 UUID.",
1905         optional => 1,
1906     },
1907     version => {
1908         type => 'string',
1909         pattern => '\S+',
1910         format_description => 'string',
1911         description => "Set SMBIOS1 version.",
1912         optional => 1,
1913     },
1914     serial => {
1915         type => 'string',
1916         pattern => '\S+',
1917         format_description => 'string',
1918         description => "Set SMBIOS1 serial number.",
1919         optional => 1,
1920     },
1921     manufacturer => {
1922         type => 'string',
1923         pattern => '\S+',
1924         format_description => 'string',
1925         description => "Set SMBIOS1 manufacturer.",
1926         optional => 1,
1927     },
1928     product => {
1929         type => 'string',
1930         pattern => '\S+',
1931         format_description => 'string',
1932         description => "Set SMBIOS1 product ID.",
1933         optional => 1,
1934     },
1935     sku => {
1936         type => 'string',
1937         pattern => '\S+',
1938         format_description => 'string',
1939         description => "Set SMBIOS1 SKU string.",
1940         optional => 1,
1941     },
1942     family => {
1943         type => 'string',
1944         pattern => '\S+',
1945         format_description => 'string',
1946         description => "Set SMBIOS1 family string.",
1947         optional => 1,
1948     },
1949 };
1950
1951 sub parse_smbios1 {
1952     my ($data) = @_;
1953
1954     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1955     warn $@ if $@;
1956     return $res;
1957 }
1958
1959 sub print_smbios1 {
1960     my ($smbios1) = @_;
1961     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1962 }
1963
1964 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1965
1966 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1967 sub verify_bootdisk {
1968     my ($value, $noerr) = @_;
1969
1970     return $value if is_valid_drivename($value);
1971
1972     return undef if $noerr;
1973
1974     die "invalid boot disk '$value'\n";
1975 }
1976
1977 sub parse_watchdog {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1983     warn $@ if $@;
1984     return $res;
1985 }
1986
1987 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1988 sub verify_usb_device {
1989     my ($value, $noerr) = @_;
1990
1991     return $value if parse_usb_device($value);
1992
1993     return undef if $noerr;
1994
1995     die "unable to parse usb device\n";
1996 }
1997
1998 # add JSON properties for create and set function
1999 sub json_config_properties {
2000     my $prop = shift;
2001
2002     foreach my $opt (keys %$confdesc) {
2003         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2004         $prop->{$opt} = $confdesc->{$opt};
2005     }
2006
2007     return $prop;
2008 }
2009
2010 sub check_type {
2011     my ($key, $value) = @_;
2012
2013     die "unknown setting '$key'\n" if !$confdesc->{$key};
2014
2015     my $type = $confdesc->{$key}->{type};
2016
2017     if (!defined($value)) {
2018         die "got undefined value\n";
2019     }
2020
2021     if ($value =~ m/[\n\r]/) {
2022         die "property contains a line feed\n";
2023     }
2024
2025     if ($type eq 'boolean') {
2026         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2027         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2028         die "type check ('boolean') failed - got '$value'\n";
2029     } elsif ($type eq 'integer') {
2030         return int($1) if $value =~ m/^(\d+)$/;
2031         die "type check ('integer') failed - got '$value'\n";
2032     } elsif ($type eq 'number') {
2033         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2034         die "type check ('number') failed - got '$value'\n";
2035     } elsif ($type eq 'string') {
2036         if (my $fmt = $confdesc->{$key}->{format}) {
2037             PVE::JSONSchema::check_format($fmt, $value);
2038             return $value;
2039         }
2040         $value =~ s/^\"(.*)\"$/$1/;
2041         return $value;
2042     } else {
2043         die "internal error"
2044     }
2045 }
2046
2047 sub check_iommu_support{
2048     #fixme : need to check IOMMU support
2049     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2050
2051     my $iommu=1;
2052     return $iommu;
2053
2054 }
2055
2056 sub touch_config {
2057     my ($vmid) = @_;
2058
2059     my $conf = PVE::QemuConfig->config_file($vmid);
2060     utime undef, undef, $conf;
2061 }
2062
2063 sub destroy_vm {
2064     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2065
2066     my $conffile = PVE::QemuConfig->config_file($vmid);
2067
2068     my $conf = PVE::QemuConfig->load_config($vmid);
2069
2070     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2071
2072     # only remove disks owned by this VM
2073     foreach_drive($conf, sub {
2074         my ($ds, $drive) = @_;
2075
2076         return if drive_is_cdrom($drive);
2077
2078         my $volid = $drive->{file};
2079
2080         return if !$volid || $volid =~ m|^/|;
2081
2082         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2083         return if !$path || !$owner || ($owner != $vmid);
2084
2085         eval {
2086             PVE::Storage::vdisk_free($storecfg, $volid);
2087         };
2088         warn "Could not remove disk '$volid', check manually: $@" if $@;
2089
2090     });
2091
2092     if ($keep_empty_config) {
2093         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2094     } else {
2095         unlink $conffile;
2096     }
2097
2098     # also remove unused disk
2099     eval {
2100         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2101
2102         eval {
2103             PVE::Storage::foreach_volid($dl, sub {
2104                 my ($volid, $sid, $volname, $d) = @_;
2105                 PVE::Storage::vdisk_free($storecfg, $volid);
2106             });
2107         };
2108         warn $@ if $@;
2109
2110     };
2111     warn $@ if $@;
2112 }
2113
2114 sub parse_vm_config {
2115     my ($filename, $raw) = @_;
2116
2117     return undef if !defined($raw);
2118
2119     my $res = {
2120         digest => Digest::SHA::sha1_hex($raw),
2121         snapshots => {},
2122         pending => {},
2123     };
2124
2125     $filename =~ m|/qemu-server/(\d+)\.conf$|
2126         || die "got strange filename '$filename'";
2127
2128     my $vmid = $1;
2129
2130     my $conf = $res;
2131     my $descr;
2132     my $section = '';
2133
2134     my @lines = split(/\n/, $raw);
2135     foreach my $line (@lines) {
2136         next if $line =~ m/^\s*$/;
2137
2138         if ($line =~ m/^\[PENDING\]\s*$/i) {
2139             $section = 'pending';
2140             if (defined($descr)) {
2141                 $descr =~ s/\s+$//;
2142                 $conf->{description} = $descr;
2143             }
2144             $descr = undef;
2145             $conf = $res->{$section} = {};
2146             next;
2147
2148         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2149             $section = $1;
2150             if (defined($descr)) {
2151                 $descr =~ s/\s+$//;
2152                 $conf->{description} = $descr;
2153             }
2154             $descr = undef;
2155             $conf = $res->{snapshots}->{$section} = {};
2156             next;
2157         }
2158
2159         if ($line =~ m/^\#(.*)\s*$/) {
2160             $descr = '' if !defined($descr);
2161             $descr .= PVE::Tools::decode_text($1) . "\n";
2162             next;
2163         }
2164
2165         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2166             $descr = '' if !defined($descr);
2167             $descr .= PVE::Tools::decode_text($2);
2168         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2169             $conf->{snapstate} = $1;
2170         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2171             my $key = $1;
2172             my $value = $2;
2173             $conf->{$key} = $value;
2174         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2175             my $value = $1;
2176             if ($section eq 'pending') {
2177                 $conf->{delete} = $value; # we parse this later
2178             } else {
2179                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2180             }
2181         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2182             my $key = $1;
2183             my $value = $2;
2184             eval { $value = check_type($key, $value); };
2185             if ($@) {
2186                 warn "vm $vmid - unable to parse value of '$key' - $@";
2187             } else {
2188                 $key = 'ide2' if $key eq 'cdrom';
2189                 my $fmt = $confdesc->{$key}->{format};
2190                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2191                     my $v = parse_drive($key, $value);
2192                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2193                         $v->{file} = $volid;
2194                         $value = print_drive($vmid, $v);
2195                     } else {
2196                         warn "vm $vmid - unable to parse value of '$key'\n";
2197                         next;
2198                     }
2199                 }
2200
2201                 $conf->{$key} = $value;
2202             }
2203         }
2204     }
2205
2206     if (defined($descr)) {
2207         $descr =~ s/\s+$//;
2208         $conf->{description} = $descr;
2209     }
2210     delete $res->{snapstate}; # just to be sure
2211
2212     return $res;
2213 }
2214
2215 sub write_vm_config {
2216     my ($filename, $conf) = @_;
2217
2218     delete $conf->{snapstate}; # just to be sure
2219
2220     if ($conf->{cdrom}) {
2221         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2222         $conf->{ide2} = $conf->{cdrom};
2223         delete $conf->{cdrom};
2224     }
2225
2226     # we do not use 'smp' any longer
2227     if ($conf->{sockets}) {
2228         delete $conf->{smp};
2229     } elsif ($conf->{smp}) {
2230         $conf->{sockets} = $conf->{smp};
2231         delete $conf->{cores};
2232         delete $conf->{smp};
2233     }
2234
2235     my $used_volids = {};
2236
2237     my $cleanup_config = sub {
2238         my ($cref, $pending, $snapname) = @_;
2239
2240         foreach my $key (keys %$cref) {
2241             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2242                 $key eq 'snapstate' || $key eq 'pending';
2243             my $value = $cref->{$key};
2244             if ($key eq 'delete') {
2245                 die "propertry 'delete' is only allowed in [PENDING]\n"
2246                     if !$pending;
2247                 # fixme: check syntax?
2248                 next;
2249             }
2250             eval { $value = check_type($key, $value); };
2251             die "unable to parse value of '$key' - $@" if $@;
2252
2253             $cref->{$key} = $value;
2254
2255             if (!$snapname && is_valid_drivename($key)) {
2256                 my $drive = parse_drive($key, $value);
2257                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2258             }
2259         }
2260     };
2261
2262     &$cleanup_config($conf);
2263
2264     &$cleanup_config($conf->{pending}, 1);
2265
2266     foreach my $snapname (keys %{$conf->{snapshots}}) {
2267         die "internal error" if $snapname eq 'pending';
2268         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2269     }
2270
2271     # remove 'unusedX' settings if we re-add a volume
2272     foreach my $key (keys %$conf) {
2273         my $value = $conf->{$key};
2274         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2275             delete $conf->{$key};
2276         }
2277     }
2278
2279     my $generate_raw_config = sub {
2280         my ($conf, $pending) = @_;
2281
2282         my $raw = '';
2283
2284         # add description as comment to top of file
2285         if (defined(my $descr = $conf->{description})) {
2286             if ($descr) {
2287                 foreach my $cl (split(/\n/, $descr)) {
2288                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2289                 }
2290             } else {
2291                 $raw .= "#\n" if $pending;
2292             }
2293         }
2294
2295         foreach my $key (sort keys %$conf) {
2296             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2297             $raw .= "$key: $conf->{$key}\n";
2298         }
2299         return $raw;
2300     };
2301
2302     my $raw = &$generate_raw_config($conf);
2303
2304     if (scalar(keys %{$conf->{pending}})){
2305         $raw .= "\n[PENDING]\n";
2306         $raw .= &$generate_raw_config($conf->{pending}, 1);
2307     }
2308
2309     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2310         $raw .= "\n[$snapname]\n";
2311         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2312     }
2313
2314     return $raw;
2315 }
2316
2317 sub load_defaults {
2318
2319     my $res = {};
2320
2321     # we use static defaults from our JSON schema configuration
2322     foreach my $key (keys %$confdesc) {
2323         if (defined(my $default = $confdesc->{$key}->{default})) {
2324             $res->{$key} = $default;
2325         }
2326     }
2327
2328     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2329     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2330
2331     return $res;
2332 }
2333
2334 sub config_list {
2335     my $vmlist = PVE::Cluster::get_vmlist();
2336     my $res = {};
2337     return $res if !$vmlist || !$vmlist->{ids};
2338     my $ids = $vmlist->{ids};
2339
2340     foreach my $vmid (keys %$ids) {
2341         my $d = $ids->{$vmid};
2342         next if !$d->{node} || $d->{node} ne $nodename;
2343         next if !$d->{type} || $d->{type} ne 'qemu';
2344         $res->{$vmid}->{exists} = 1;
2345     }
2346     return $res;
2347 }
2348
2349 # test if VM uses local resources (to prevent migration)
2350 sub check_local_resources {
2351     my ($conf, $noerr) = @_;
2352
2353     my $loc_res = 0;
2354
2355     $loc_res = 1 if $conf->{hostusb}; # old syntax
2356     $loc_res = 1 if $conf->{hostpci}; # old syntax
2357
2358     foreach my $k (keys %$conf) {
2359         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2360         # sockets are safe: they will recreated be on the target side post-migrate
2361         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2362         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2363     }
2364
2365     die "VM uses local resources\n" if $loc_res && !$noerr;
2366
2367     return $loc_res;
2368 }
2369
2370 # check if used storages are available on all nodes (use by migrate)
2371 sub check_storage_availability {
2372     my ($storecfg, $conf, $node) = @_;
2373
2374     foreach_drive($conf, sub {
2375         my ($ds, $drive) = @_;
2376
2377         my $volid = $drive->{file};
2378         return if !$volid;
2379
2380         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2381         return if !$sid;
2382
2383         # check if storage is available on both nodes
2384         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2385         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2386    });
2387 }
2388
2389 # list nodes where all VM images are available (used by has_feature API)
2390 sub shared_nodes {
2391     my ($conf, $storecfg) = @_;
2392
2393     my $nodelist = PVE::Cluster::get_nodelist();
2394     my $nodehash = { map { $_ => 1 } @$nodelist };
2395     my $nodename = PVE::INotify::nodename();
2396
2397     foreach_drive($conf, sub {
2398         my ($ds, $drive) = @_;
2399
2400         my $volid = $drive->{file};
2401         return if !$volid;
2402
2403         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2404         if ($storeid) {
2405             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2406             if ($scfg->{disable}) {
2407                 $nodehash = {};
2408             } elsif (my $avail = $scfg->{nodes}) {
2409                 foreach my $node (keys %$nodehash) {
2410                     delete $nodehash->{$node} if !$avail->{$node};
2411                 }
2412             } elsif (!$scfg->{shared}) {
2413                 foreach my $node (keys %$nodehash) {
2414                     delete $nodehash->{$node} if $node ne $nodename
2415                 }
2416             }
2417         }
2418     });
2419
2420     return $nodehash
2421 }
2422
2423 sub check_cmdline {
2424     my ($pidfile, $pid) = @_;
2425
2426     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2427     if (defined($fh)) {
2428         my $line = <$fh>;
2429         $fh->close;
2430         return undef if !$line;
2431         my @param = split(/\0/, $line);
2432
2433         my $cmd = $param[0];
2434         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2435
2436         for (my $i = 0; $i < scalar (@param); $i++) {
2437             my $p = $param[$i];
2438             next if !$p;
2439             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2440                 my $p = $param[$i+1];
2441                 return 1 if $p && ($p eq $pidfile);
2442                 return undef;
2443             }
2444         }
2445     }
2446     return undef;
2447 }
2448
2449 sub check_running {
2450     my ($vmid, $nocheck, $node) = @_;
2451
2452     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2453
2454     die "unable to find configuration file for VM $vmid - no such machine\n"
2455         if !$nocheck && ! -f $filename;
2456
2457     my $pidfile = pidfile_name($vmid);
2458
2459     if (my $fd = IO::File->new("<$pidfile")) {
2460         my $st = stat($fd);
2461         my $line = <$fd>;
2462         close($fd);
2463
2464         my $mtime = $st->mtime;
2465         if ($mtime > time()) {
2466             warn "file '$filename' modified in future\n";
2467         }
2468
2469         if ($line =~ m/^(\d+)$/) {
2470             my $pid = $1;
2471             if (check_cmdline($pidfile, $pid)) {
2472                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2473                     return $pid;
2474                 }
2475             }
2476         }
2477     }
2478
2479     return undef;
2480 }
2481
2482 sub vzlist {
2483
2484     my $vzlist = config_list();
2485
2486     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2487
2488     while (defined(my $de = $fd->read)) {
2489         next if $de !~ m/^(\d+)\.pid$/;
2490         my $vmid = $1;
2491         next if !defined($vzlist->{$vmid});
2492         if (my $pid = check_running($vmid)) {
2493             $vzlist->{$vmid}->{pid} = $pid;
2494         }
2495     }
2496
2497     return $vzlist;
2498 }
2499
2500 sub disksize {
2501     my ($storecfg, $conf) = @_;
2502
2503     my $bootdisk = $conf->{bootdisk};
2504     return undef if !$bootdisk;
2505     return undef if !is_valid_drivename($bootdisk);
2506
2507     return undef if !$conf->{$bootdisk};
2508
2509     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2510     return undef if !defined($drive);
2511
2512     return undef if drive_is_cdrom($drive);
2513
2514     my $volid = $drive->{file};
2515     return undef if !$volid;
2516
2517     return $drive->{size};
2518 }
2519
2520 my $last_proc_pid_stat;
2521
2522 # get VM status information
2523 # This must be fast and should not block ($full == false)
2524 # We only query KVM using QMP if $full == true (this can be slow)
2525 sub vmstatus {
2526     my ($opt_vmid, $full) = @_;
2527
2528     my $res = {};
2529
2530     my $storecfg = PVE::Storage::config();
2531
2532     my $list = vzlist();
2533     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2534
2535     my $cpucount = $cpuinfo->{cpus} || 1;
2536
2537     foreach my $vmid (keys %$list) {
2538         next if $opt_vmid && ($vmid ne $opt_vmid);
2539
2540         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2541         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2542
2543         my $d = {};
2544         $d->{pid} = $list->{$vmid}->{pid};
2545
2546         # fixme: better status?
2547         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2548
2549         my $size = disksize($storecfg, $conf);
2550         if (defined($size)) {
2551             $d->{disk} = 0; # no info available
2552             $d->{maxdisk} = $size;
2553         } else {
2554             $d->{disk} = 0;
2555             $d->{maxdisk} = 0;
2556         }
2557
2558         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2559         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2560         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2561
2562         $d->{name} = $conf->{name} || "VM $vmid";
2563         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2564
2565         if ($conf->{balloon}) {
2566             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2567             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2568         }
2569
2570         $d->{uptime} = 0;
2571         $d->{cpu} = 0;
2572         $d->{mem} = 0;
2573
2574         $d->{netout} = 0;
2575         $d->{netin} = 0;
2576
2577         $d->{diskread} = 0;
2578         $d->{diskwrite} = 0;
2579
2580         $d->{template} = PVE::QemuConfig->is_template($conf);
2581
2582         $res->{$vmid} = $d;
2583     }
2584
2585     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2586     foreach my $dev (keys %$netdev) {
2587         next if $dev !~ m/^tap([1-9]\d*)i/;
2588         my $vmid = $1;
2589         my $d = $res->{$vmid};
2590         next if !$d;
2591
2592         $d->{netout} += $netdev->{$dev}->{receive};
2593         $d->{netin} += $netdev->{$dev}->{transmit};
2594
2595         if ($full) {
2596             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2597             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2598         }
2599
2600     }
2601
2602     my $ctime = gettimeofday;
2603
2604     foreach my $vmid (keys %$list) {
2605
2606         my $d = $res->{$vmid};
2607         my $pid = $d->{pid};
2608         next if !$pid;
2609
2610         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2611         next if !$pstat; # not running
2612
2613         my $used = $pstat->{utime} + $pstat->{stime};
2614
2615         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2616
2617         if ($pstat->{vsize}) {
2618             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2619         }
2620
2621         my $old = $last_proc_pid_stat->{$pid};
2622         if (!$old) {
2623             $last_proc_pid_stat->{$pid} = {
2624                 time => $ctime,
2625                 used => $used,
2626                 cpu => 0,
2627             };
2628             next;
2629         }
2630
2631         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2632
2633         if ($dtime > 1000) {
2634             my $dutime = $used -  $old->{used};
2635
2636             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2637             $last_proc_pid_stat->{$pid} = {
2638                 time => $ctime,
2639                 used => $used,
2640                 cpu => $d->{cpu},
2641             };
2642         } else {
2643             $d->{cpu} = $old->{cpu};
2644         }
2645     }
2646
2647     return $res if !$full;
2648
2649     my $qmpclient = PVE::QMPClient->new();
2650
2651     my $ballooncb = sub {
2652         my ($vmid, $resp) = @_;
2653
2654         my $info = $resp->{'return'};
2655         return if !$info->{max_mem};
2656
2657         my $d = $res->{$vmid};
2658
2659         # use memory assigned to VM
2660         $d->{maxmem} = $info->{max_mem};
2661         $d->{balloon} = $info->{actual};
2662
2663         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2664             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2665             $d->{freemem} = $info->{free_mem};
2666         }
2667
2668         $d->{ballooninfo} = $info;
2669     };
2670
2671     my $blockstatscb = sub {
2672         my ($vmid, $resp) = @_;
2673         my $data = $resp->{'return'} || [];
2674         my $totalrdbytes = 0;
2675         my $totalwrbytes = 0;
2676
2677         for my $blockstat (@$data) {
2678             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2679             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2680
2681             $blockstat->{device} =~ s/drive-//;
2682             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2683         }
2684         $res->{$vmid}->{diskread} = $totalrdbytes;
2685         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2686     };
2687
2688     my $statuscb = sub {
2689         my ($vmid, $resp) = @_;
2690
2691         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2692         # this fails if ballon driver is not loaded, so this must be
2693         # the last commnand (following command are aborted if this fails).
2694         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2695
2696         my $status = 'unknown';
2697         if (!defined($status = $resp->{'return'}->{status})) {
2698             warn "unable to get VM status\n";
2699             return;
2700         }
2701
2702         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2703     };
2704
2705     foreach my $vmid (keys %$list) {
2706         next if $opt_vmid && ($vmid ne $opt_vmid);
2707         next if !$res->{$vmid}->{pid}; # not running
2708         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2709     }
2710
2711     $qmpclient->queue_execute(undef, 2);
2712
2713     foreach my $vmid (keys %$list) {
2714         next if $opt_vmid && ($vmid ne $opt_vmid);
2715         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2716     }
2717
2718     return $res;
2719 }
2720
2721 sub foreach_drive {
2722     my ($conf, $func, @param) = @_;
2723
2724     foreach my $ds (valid_drive_names()) {
2725         next if !defined($conf->{$ds});
2726
2727         my $drive = parse_drive($ds, $conf->{$ds});
2728         next if !$drive;
2729
2730         &$func($ds, $drive, @param);
2731     }
2732 }
2733
2734 sub foreach_volid {
2735     my ($conf, $func, @param) = @_;
2736
2737     my $volhash = {};
2738
2739     my $test_volid = sub {
2740         my ($volid, $is_cdrom) = @_;
2741
2742         return if !$volid;
2743
2744         $volhash->{$volid} = $is_cdrom || 0;
2745     };
2746
2747     foreach_drive($conf, sub {
2748         my ($ds, $drive) = @_;
2749         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2750     });
2751
2752     foreach my $snapname (keys %{$conf->{snapshots}}) {
2753         my $snap = $conf->{snapshots}->{$snapname};
2754         &$test_volid($snap->{vmstate}, 0);
2755         foreach_drive($snap, sub {
2756             my ($ds, $drive) = @_;
2757             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2758         });
2759     }
2760
2761     foreach my $volid (keys %$volhash) {
2762         &$func($volid, $volhash->{$volid}, @param);
2763     }
2764 }
2765
2766 sub vga_conf_has_spice {
2767     my ($vga) = @_;
2768
2769     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2770
2771     return $1 || 1;
2772 }
2773
2774 sub config_to_command {
2775     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2776
2777     my $cmd = [];
2778     my $globalFlags = [];
2779     my $machineFlags = [];
2780     my $rtcFlags = [];
2781     my $cpuFlags = [];
2782     my $devices = [];
2783     my $pciaddr = '';
2784     my $bridges = {};
2785     my $kvmver = kvm_user_version();
2786     my $vernum = 0; # unknown
2787     my $ostype = $conf->{ostype};
2788     my $winversion = windows_version($ostype);
2789
2790     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2791         $vernum = $1*1000000+$2*1000;
2792     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2793         $vernum = $1*1000000+$2*1000+$3;
2794     }
2795
2796     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2797
2798     my $have_ovz = -f '/proc/vz/vestat';
2799
2800     my $q35 = machine_type_is_q35($conf);
2801     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2802     my $machine_type = $forcemachine || $conf->{machine};
2803     my $use_old_bios_files = undef;
2804     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2805
2806     my $cpuunits = defined($conf->{cpuunits}) ?
2807             $conf->{cpuunits} : $defaults->{cpuunits};
2808
2809     push @$cmd, '/usr/bin/kvm';
2810
2811     push @$cmd, '-id', $vmid;
2812
2813     my $use_virtio = 0;
2814
2815     my $qmpsocket = qmp_socket($vmid);
2816     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2817     push @$cmd, '-mon', "chardev=qmp,mode=control";
2818
2819
2820     push @$cmd, '-pidfile' , pidfile_name($vmid);
2821
2822     push @$cmd, '-daemonize';
2823
2824     if ($conf->{smbios1}) {
2825         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2826     }
2827
2828     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2829         my $ovmfbase;
2830
2831         # prefer the OVMF_CODE variant
2832         if (-f $OVMF_CODE) {
2833             $ovmfbase = $OVMF_CODE;
2834         } elsif (-f $OVMF_IMG) {
2835             $ovmfbase = $OVMF_IMG;
2836         }
2837
2838         die "no uefi base img found\n" if !$ovmfbase;
2839         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2840
2841         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2842             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2843             my $format = $d->{format} // 'raw';
2844             my $path;
2845             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2846             if ($storeid) {
2847                 $path = PVE::Storage::path($storecfg, $d->{file});
2848                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2849                 $format = qemu_img_format($scfg, $volname);
2850             } else {
2851                 $path = $d->{file};
2852                 $format = "raw";
2853             }
2854             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2855         } elsif ($ovmfbase eq $OVMF_CODE) {
2856             warn "using uefi without permanent efivars disk\n";
2857             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2858             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2859             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2860         } else {
2861             # if the base img is not OVMF_CODE, we do not have to bother
2862             # to create/use a vars image, since it will not be used anyway
2863             # this can only happen if someone manually deletes the OVMF_CODE image
2864             # or has an old pve-qemu-kvm version installed.
2865             # both should not happen, but we ignore it here
2866         }
2867     }
2868
2869
2870     # add usb controllers
2871     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2872     push @$devices, @usbcontrollers if @usbcontrollers;
2873     my $vga = $conf->{vga};
2874
2875     my $qxlnum = vga_conf_has_spice($vga);
2876     $vga = 'qxl' if $qxlnum;
2877
2878     if (!$vga) {
2879         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2880     }
2881
2882     # enable absolute mouse coordinates (needed by vnc)
2883     my $tablet;
2884     if (defined($conf->{tablet})) {
2885         $tablet = $conf->{tablet};
2886     } else {
2887         $tablet = $defaults->{tablet};
2888         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2889         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2890     }
2891
2892     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2893
2894     my $kvm_off = 0;
2895     my $gpu_passthrough;
2896
2897     # host pci devices
2898     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2899         my $d = parse_hostpci($conf->{"hostpci$i"});
2900         next if !$d;
2901
2902         my $pcie = $d->{pcie};
2903         if($pcie){
2904             die "q35 machine model is not enabled" if !$q35;
2905             $pciaddr = print_pcie_addr("hostpci$i");
2906         }else{
2907             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2908         }
2909
2910         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2911         my $xvga = '';
2912         if ($d->{'x-vga'}) {
2913             $xvga = ',x-vga=on';
2914             $kvm_off = 1;
2915             $vga = 'none';
2916             $gpu_passthrough = 1;
2917
2918             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2919                 $xvga = "";
2920             }
2921         }
2922         my $pcidevices = $d->{pciid};
2923         my $multifunction = 1 if @$pcidevices > 1;
2924
2925         my $j=0;
2926         foreach my $pcidevice (@$pcidevices) {
2927
2928             my $id = "hostpci$i";
2929             $id .= ".$j" if $multifunction;
2930             my $addr = $pciaddr;
2931             $addr .= ".$j" if $multifunction;
2932             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2933
2934             if($j == 0){
2935                 $devicestr .= "$rombar$xvga";
2936                 $devicestr .= ",multifunction=on" if $multifunction;
2937             }
2938
2939             push @$devices, '-device', $devicestr;
2940             $j++;
2941         }
2942     }
2943
2944     # usb devices
2945     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2946     push @$devices, @usbdevices if @usbdevices;
2947     # serial devices
2948     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2949         if (my $path = $conf->{"serial$i"}) {
2950             if ($path eq 'socket') {
2951                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2952                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2953                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2954             } else {
2955                 die "no such serial device\n" if ! -c $path;
2956                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2957                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2958             }
2959         }
2960     }
2961
2962     # parallel devices
2963     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2964         if (my $path = $conf->{"parallel$i"}) {
2965             die "no such parallel device\n" if ! -c $path;
2966             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2967             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2968             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2969         }
2970     }
2971
2972     my $vmname = $conf->{name} || "vm$vmid";
2973
2974     push @$cmd, '-name', $vmname;
2975
2976     my $sockets = 1;
2977     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2978     $sockets = $conf->{sockets} if  $conf->{sockets};
2979
2980     my $cores = $conf->{cores} || 1;
2981
2982     my $maxcpus = $sockets * $cores;
2983
2984     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2985
2986     my $allowed_vcpus = $cpuinfo->{cpus};
2987
2988     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2989         if ($allowed_vcpus < $maxcpus);
2990
2991     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2992
2993         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2994         for (my $i = 2; $i <= $vcpus; $i++)  {
2995             my $cpustr = print_cpu_device($conf,$i);
2996             push @$cmd, '-device', $cpustr;
2997         }
2998
2999     } else {
3000
3001         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3002     }
3003     push @$cmd, '-nodefaults';
3004
3005     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3006
3007     my $bootindex_hash = {};
3008     my $i = 1;
3009     foreach my $o (split(//, $bootorder)) {
3010         $bootindex_hash->{$o} = $i*100;
3011         $i++;
3012     }
3013
3014     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3015
3016     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3017
3018     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3019
3020     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3021
3022     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3023         my $socket = vnc_socket($vmid);
3024         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3025     } else {
3026         push @$cmd, '-nographic';
3027     }
3028
3029     # time drift fix
3030     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3031
3032     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3033     my $useLocaltime = $conf->{localtime};
3034
3035     if ($winversion >= 5) { # windows
3036         $useLocaltime = 1 if !defined($conf->{localtime});
3037
3038         # use time drift fix when acpi is enabled
3039         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3040             $tdf = 1 if !defined($conf->{tdf});
3041         }
3042     }
3043
3044     if ($winversion >= 6) {
3045         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3046         push @$cmd, '-no-hpet';
3047     }
3048
3049     push @$rtcFlags, 'driftfix=slew' if $tdf;
3050
3051     if ($nokvm) {
3052         push @$machineFlags, 'accel=tcg';
3053     } else {
3054         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3055     }
3056
3057     if ($machine_type) {
3058         push @$machineFlags, "type=${machine_type}";
3059     }
3060
3061     if ($conf->{startdate}) {
3062         push @$rtcFlags, "base=$conf->{startdate}";
3063     } elsif ($useLocaltime) {
3064         push @$rtcFlags, 'base=localtime';
3065     }
3066
3067     my $cpu = $nokvm ? "qemu64" : "kvm64";
3068     if (my $cputype = $conf->{cpu}) {
3069         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3070             or die "Cannot parse cpu description: $cputype\n";
3071         $cpu = $cpuconf->{cputype};
3072         $kvm_off = 1 if $cpuconf->{hidden};
3073     }
3074
3075     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3076
3077     push @$cpuFlags , '-x2apic'
3078         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3079
3080     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3081
3082     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3083
3084     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3085
3086         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3087         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3088     }
3089
3090     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3091
3092     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3093
3094     push @$cpuFlags, 'kvm=off' if $kvm_off;
3095
3096     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3097         die "internal error"; # should not happen
3098
3099     push @$cpuFlags, "vendor=${cpu_vendor}"
3100         if $cpu_vendor ne 'default';
3101
3102     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3103
3104     push @$cmd, '-cpu', $cpu;
3105
3106     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3107
3108     push @$cmd, '-S' if $conf->{freeze};
3109
3110     # set keyboard layout
3111     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3112     push @$cmd, '-k', $kb if $kb;
3113
3114     # enable sound
3115     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3116     #push @$cmd, '-soundhw', 'es1370';
3117     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3118
3119     if($conf->{agent}) {
3120         my $qgasocket = qmp_socket($vmid, 1);
3121         my $pciaddr = print_pci_addr("qga0", $bridges);
3122         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3123         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3124         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3125     }
3126
3127     my $spice_port;
3128
3129     if ($qxlnum) {
3130         if ($qxlnum > 1) {
3131             if ($winversion){
3132                 for(my $i = 1; $i < $qxlnum; $i++){
3133                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3134                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3135                 }
3136             } else {
3137                 # assume other OS works like Linux
3138                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3139                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3140             }
3141         }
3142
3143         my $pciaddr = print_pci_addr("spice", $bridges);
3144
3145         my $nodename = PVE::INotify::nodename();
3146         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3147         $spice_port = PVE::Tools::next_spice_port($pfamily);
3148
3149         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3150
3151         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3152         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3153         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3154     }
3155
3156     # enable balloon by default, unless explicitly disabled
3157     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3158         $pciaddr = print_pci_addr("balloon0", $bridges);
3159         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3160     }
3161
3162     if ($conf->{watchdog}) {
3163         my $wdopts = parse_watchdog($conf->{watchdog});
3164         $pciaddr = print_pci_addr("watchdog", $bridges);
3165         my $watchdog = $wdopts->{model} || 'i6300esb';
3166         push @$devices, '-device', "$watchdog$pciaddr";
3167         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3168     }
3169
3170     my $vollist = [];
3171     my $scsicontroller = {};
3172     my $ahcicontroller = {};
3173     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3174
3175     # Add iscsi initiator name if available
3176     if (my $initiator = get_initiator_name()) {
3177         push @$devices, '-iscsi', "initiator-name=$initiator";
3178     }
3179
3180     foreach_drive($conf, sub {
3181         my ($ds, $drive) = @_;
3182
3183         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3184             push @$vollist, $drive->{file};
3185         }
3186
3187         $use_virtio = 1 if $ds =~ m/^virtio/;
3188
3189         if (drive_is_cdrom ($drive)) {
3190             if ($bootindex_hash->{d}) {
3191                 $drive->{bootindex} = $bootindex_hash->{d};
3192                 $bootindex_hash->{d} += 1;
3193             }
3194         } else {
3195             if ($bootindex_hash->{c}) {
3196                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3197                 $bootindex_hash->{c} += 1;
3198             }
3199         }
3200
3201         if($drive->{interface} eq 'virtio'){
3202            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3203         }
3204
3205         if ($drive->{interface} eq 'scsi') {
3206
3207             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3208
3209             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3210             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3211
3212             my $iothread = '';
3213             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3214                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3215                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3216             } elsif ($drive->{iothread}) {
3217                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3218             }
3219
3220             my $queues = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3222                 $queues = ",num_queues=$drive->{queues}";
3223             }
3224
3225             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3226             $scsicontroller->{$controller}=1;
3227         }
3228
3229         if ($drive->{interface} eq 'sata') {
3230            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3231            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3232            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3233            $ahcicontroller->{$controller}=1;
3234         }
3235
3236         if ($drive->{interface} eq 'efidisk') {
3237             # this will be added somewhere else
3238             return;
3239         }
3240
3241         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3242         push @$devices, '-drive',$drive_cmd;
3243         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3244     });
3245
3246     for (my $i = 0; $i < $MAX_NETS; $i++) {
3247          next if !$conf->{"net$i"};
3248          my $d = parse_net($conf->{"net$i"});
3249          next if !$d;
3250
3251          $use_virtio = 1 if $d->{model} eq 'virtio';
3252
3253          if ($bootindex_hash->{n}) {
3254             $d->{bootindex} = $bootindex_hash->{n};
3255             $bootindex_hash->{n} += 1;
3256          }
3257
3258          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3259          push @$devices, '-netdev', $netdevfull;
3260
3261          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3262          push @$devices, '-device', $netdevicefull;
3263     }
3264
3265     if (!$q35) {
3266         # add pci bridges
3267         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3268            $bridges->{1} = 1;
3269            $bridges->{2} = 1;
3270         }
3271
3272         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3273
3274         while (my ($k, $v) = each %$bridges) {
3275             $pciaddr = print_pci_addr("pci.$k");
3276             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3277         }
3278     }
3279
3280     # add custom args
3281     if ($conf->{args}) {
3282         my $aa = PVE::Tools::split_args($conf->{args});
3283         push @$cmd, @$aa;
3284     }
3285
3286     push @$cmd, @$devices;
3287     push @$cmd, '-rtc', join(',', @$rtcFlags)
3288         if scalar(@$rtcFlags);
3289     push @$cmd, '-machine', join(',', @$machineFlags)
3290         if scalar(@$machineFlags);
3291     push @$cmd, '-global', join(',', @$globalFlags)
3292         if scalar(@$globalFlags);
3293
3294     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3295 }
3296
3297 sub vnc_socket {
3298     my ($vmid) = @_;
3299     return "${var_run_tmpdir}/$vmid.vnc";
3300 }
3301
3302 sub spice_port {
3303     my ($vmid) = @_;
3304
3305     my $res = vm_mon_cmd($vmid, 'query-spice');
3306
3307     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3308 }
3309
3310 sub qmp_socket {
3311     my ($vmid, $qga) = @_;
3312     my $sockettype = $qga ? 'qga' : 'qmp';
3313     return "${var_run_tmpdir}/$vmid.$sockettype";
3314 }
3315
3316 sub pidfile_name {
3317     my ($vmid) = @_;
3318     return "${var_run_tmpdir}/$vmid.pid";
3319 }
3320
3321 sub vm_devices_list {
3322     my ($vmid) = @_;
3323
3324     my $res = vm_mon_cmd($vmid, 'query-pci');
3325     my $devices = {};
3326     foreach my $pcibus (@$res) {
3327         foreach my $device (@{$pcibus->{devices}}) {
3328             next if !$device->{'qdev_id'};
3329             if ($device->{'pci_bridge'}) {
3330                 $devices->{$device->{'qdev_id'}} = 1;
3331                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3332                     next if !$bridge_device->{'qdev_id'};
3333                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3334                     $devices->{$device->{'qdev_id'}}++;
3335                 }
3336             } else {
3337                 $devices->{$device->{'qdev_id'}} = 1;
3338             }
3339         }
3340     }
3341
3342     my $resblock = vm_mon_cmd($vmid, 'query-block');
3343     foreach my $block (@$resblock) {
3344         if($block->{device} =~ m/^drive-(\S+)/){
3345                 $devices->{$1} = 1;
3346         }
3347     }
3348
3349     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3350     foreach my $mice (@$resmice) {
3351         if ($mice->{name} eq 'QEMU HID Tablet') {
3352             $devices->{tablet} = 1;
3353             last;
3354         }
3355     }
3356
3357     # for usb devices there is no query-usb
3358     # but we can iterate over the entries in
3359     # qom-list path=/machine/peripheral
3360     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3361     foreach my $per (@$resperipheral) {
3362         if ($per->{name} =~ m/^usb\d+$/) {
3363             $devices->{$per->{name}} = 1;
3364         }
3365     }
3366
3367     return $devices;
3368 }
3369
3370 sub vm_deviceplug {
3371     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3372
3373     my $q35 = machine_type_is_q35($conf);
3374
3375     my $devices_list = vm_devices_list($vmid);
3376     return 1 if defined($devices_list->{$deviceid});
3377
3378     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3379
3380     if ($deviceid eq 'tablet') {
3381
3382         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3383
3384     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3385
3386         die "usb hotplug currently not reliable\n";
3387         # since we can't reliably hot unplug all added usb devices
3388         # and usb passthrough disables live migration
3389         # we disable usb hotplugging for now
3390         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3391
3392     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3393
3394         qemu_iothread_add($vmid, $deviceid, $device);
3395
3396         qemu_driveadd($storecfg, $vmid, $device);
3397         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3398
3399         qemu_deviceadd($vmid, $devicefull);
3400         eval { qemu_deviceaddverify($vmid, $deviceid); };
3401         if (my $err = $@) {
3402             eval { qemu_drivedel($vmid, $deviceid); };
3403             warn $@ if $@;
3404             die $err;
3405         }
3406
3407     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3408
3409
3410         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3411         my $pciaddr = print_pci_addr($deviceid);
3412         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3413
3414         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3417             qemu_iothread_add($vmid, $deviceid, $device);
3418             $devicefull .= ",iothread=iothread-$deviceid";
3419         }
3420
3421         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3422             $devicefull .= ",num_queues=$device->{queues}";
3423         }
3424
3425         qemu_deviceadd($vmid, $devicefull);
3426         qemu_deviceaddverify($vmid, $deviceid);
3427
3428     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3429
3430         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3431         qemu_driveadd($storecfg, $vmid, $device);
3432
3433         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3434         eval { qemu_deviceadd($vmid, $devicefull); };
3435         if (my $err = $@) {
3436             eval { qemu_drivedel($vmid, $deviceid); };
3437             warn $@ if $@;
3438             die $err;
3439         }
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3444
3445         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3446         my $use_old_bios_files = undef;
3447         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3448
3449         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3450         qemu_deviceadd($vmid, $netdevicefull);
3451         eval { qemu_deviceaddverify($vmid, $deviceid); };
3452         if (my $err = $@) {
3453             eval { qemu_netdevdel($vmid, $deviceid); };
3454             warn $@ if $@;
3455             die $err;
3456         }
3457
3458     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3459
3460         my $bridgeid = $2;
3461         my $pciaddr = print_pci_addr($deviceid);
3462         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3463
3464         qemu_deviceadd($vmid, $devicefull);
3465         qemu_deviceaddverify($vmid, $deviceid);
3466
3467     } else {
3468         die "can't hotplug device '$deviceid'\n";
3469     }
3470
3471     return 1;
3472 }
3473
3474 # fixme: this should raise exceptions on error!
3475 sub vm_deviceunplug {
3476     my ($vmid, $conf, $deviceid) = @_;
3477
3478     my $devices_list = vm_devices_list($vmid);
3479     return 1 if !defined($devices_list->{$deviceid});
3480
3481     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3482
3483     if ($deviceid eq 'tablet') {
3484
3485         qemu_devicedel($vmid, $deviceid);
3486
3487     } elsif ($deviceid =~ m/^usb\d+$/) {
3488
3489         die "usb hotplug currently not reliable\n";
3490         # when unplugging usb devices this way,
3491         # there may be remaining usb controllers/hubs
3492         # so we disable it for now
3493         qemu_devicedel($vmid, $deviceid);
3494         qemu_devicedelverify($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3497
3498         qemu_devicedel($vmid, $deviceid);
3499         qemu_devicedelverify($vmid, $deviceid);
3500         qemu_drivedel($vmid, $deviceid);
3501         qemu_iothread_del($conf, $vmid, $deviceid);
3502
3503     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3504
3505         qemu_devicedel($vmid, $deviceid);
3506         qemu_devicedelverify($vmid, $deviceid);
3507         qemu_iothread_del($conf, $vmid, $deviceid);
3508
3509     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3510
3511         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3512         my $device = parse_drive($deviceid, $conf->{$deviceid});
3513         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3514
3515         qemu_devicedel($vmid, $deviceid);
3516         qemu_drivedel($vmid, $deviceid);
3517         qemu_deletescsihw($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_netdevdel($vmid, $deviceid);
3524
3525     } else {
3526         die "can't unplug device '$deviceid'\n";
3527     }
3528
3529     return 1;
3530 }
3531
3532 sub qemu_deviceadd {
3533     my ($vmid, $devicefull) = @_;
3534
3535     $devicefull = "driver=".$devicefull;
3536     my %options =  split(/[=,]/, $devicefull);
3537
3538     vm_mon_cmd($vmid, "device_add" , %options);
3539 }
3540
3541 sub qemu_devicedel {
3542     my ($vmid, $deviceid) = @_;
3543
3544     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3545 }
3546
3547 sub qemu_iothread_add {
3548     my($vmid, $deviceid, $device) = @_;
3549
3550     if ($device->{iothread}) {
3551         my $iothreads = vm_iothreads_list($vmid);
3552         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3553     }
3554 }
3555
3556 sub qemu_iothread_del {
3557     my($conf, $vmid, $deviceid) = @_;
3558
3559     my $device = parse_drive($deviceid, $conf->{$deviceid});
3560     if ($device->{iothread}) {
3561         my $iothreads = vm_iothreads_list($vmid);
3562         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3563     }
3564 }
3565
3566 sub qemu_objectadd {
3567     my($vmid, $objectid, $qomtype) = @_;
3568
3569     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_objectdel {
3575     my($vmid, $objectid) = @_;
3576
3577     vm_mon_cmd($vmid, "object-del", id => $objectid);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_driveadd {
3583     my ($storecfg, $vmid, $device) = @_;
3584
3585     my $drive = print_drive_full($storecfg, $vmid, $device);
3586     $drive =~ s/\\/\\\\/g;
3587     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3588
3589     # If the command succeeds qemu prints: "OK"
3590     return 1 if $ret =~ m/OK/s;
3591
3592     die "adding drive failed: $ret\n";
3593 }
3594
3595 sub qemu_drivedel {
3596     my($vmid, $deviceid) = @_;
3597
3598     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3599     $ret =~ s/^\s+//;
3600
3601     return 1 if $ret eq "";
3602
3603     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3604     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3605
3606     die "deleting drive $deviceid failed : $ret\n";
3607 }
3608
3609 sub qemu_deviceaddverify {
3610     my ($vmid, $deviceid) = @_;
3611
3612     for (my $i = 0; $i <= 5; $i++) {
3613          my $devices_list = vm_devices_list($vmid);
3614          return 1 if defined($devices_list->{$deviceid});
3615          sleep 1;
3616     }
3617
3618     die "error on hotplug device '$deviceid'\n";
3619 }
3620
3621
3622 sub qemu_devicedelverify {
3623     my ($vmid, $deviceid) = @_;
3624
3625     # need to verify that the device is correctly removed as device_del
3626     # is async and empty return is not reliable
3627
3628     for (my $i = 0; $i <= 5; $i++) {
3629          my $devices_list = vm_devices_list($vmid);
3630          return 1 if !defined($devices_list->{$deviceid});
3631          sleep 1;
3632     }
3633
3634     die "error on hot-unplugging device '$deviceid'\n";
3635 }
3636
3637 sub qemu_findorcreatescsihw {
3638     my ($storecfg, $conf, $vmid, $device) = @_;
3639
3640     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3641
3642     my $scsihwid="$controller_prefix$controller";
3643     my $devices_list = vm_devices_list($vmid);
3644
3645     if(!defined($devices_list->{$scsihwid})) {
3646         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3647     }
3648
3649     return 1;
3650 }
3651
3652 sub qemu_deletescsihw {
3653     my ($conf, $vmid, $opt) = @_;
3654
3655     my $device = parse_drive($opt, $conf->{$opt});
3656
3657     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3658         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3659         return 1;
3660     }
3661
3662     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3663
3664     my $devices_list = vm_devices_list($vmid);
3665     foreach my $opt (keys %{$devices_list}) {
3666         if (PVE::QemuServer::is_valid_drivename($opt)) {
3667             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3668             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3669                 return 1;
3670             }
3671         }
3672     }
3673
3674     my $scsihwid="scsihw$controller";
3675
3676     vm_deviceunplug($vmid, $conf, $scsihwid);
3677
3678     return 1;
3679 }
3680
3681 sub qemu_add_pci_bridge {
3682     my ($storecfg, $conf, $vmid, $device) = @_;
3683
3684     my $bridges = {};
3685
3686     my $bridgeid;
3687
3688     print_pci_addr($device, $bridges);
3689
3690     while (my ($k, $v) = each %$bridges) {
3691         $bridgeid = $k;
3692     }
3693     return 1 if !defined($bridgeid) || $bridgeid < 1;
3694
3695     my $bridge = "pci.$bridgeid";
3696     my $devices_list = vm_devices_list($vmid);
3697
3698     if (!defined($devices_list->{$bridge})) {
3699         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3700     }
3701
3702     return 1;
3703 }
3704
3705 sub qemu_set_link_status {
3706     my ($vmid, $device, $up) = @_;
3707
3708     vm_mon_cmd($vmid, "set_link", name => $device,
3709                up => $up ? JSON::true : JSON::false);
3710 }
3711
3712 sub qemu_netdevadd {
3713     my ($vmid, $conf, $device, $deviceid) = @_;
3714
3715     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3716     my %options =  split(/[=,]/, $netdev);
3717
3718     vm_mon_cmd($vmid, "netdev_add",  %options);
3719     return 1;
3720 }
3721
3722 sub qemu_netdevdel {
3723     my ($vmid, $deviceid) = @_;
3724
3725     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3726 }
3727
3728 sub qemu_usb_hotplug {
3729     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3730
3731     return if !$device;
3732
3733     # remove the old one first
3734     vm_deviceunplug($vmid, $conf, $deviceid);
3735
3736     # check if xhci controller is necessary and available
3737     if ($device->{usb3}) {
3738
3739         my $devicelist = vm_devices_list($vmid);
3740
3741         if (!$devicelist->{xhci}) {
3742             my $pciaddr = print_pci_addr("xhci");
3743             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3744         }
3745     }
3746     my $d = parse_usb_device($device->{host});
3747     $d->{usb3} = $device->{usb3};
3748
3749     # add the new one
3750     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3751 }
3752
3753 sub qemu_cpu_hotplug {
3754     my ($vmid, $conf, $vcpus) = @_;
3755
3756     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3757
3758     my $sockets = 1;
3759     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3760     $sockets = $conf->{sockets} if  $conf->{sockets};
3761     my $cores = $conf->{cores} || 1;
3762     my $maxcpus = $sockets * $cores;
3763
3764     $vcpus = $maxcpus if !$vcpus;
3765
3766     die "you can't add more vcpus than maxcpus\n"
3767         if $vcpus > $maxcpus;
3768
3769     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3770
3771     if ($vcpus < $currentvcpus) {
3772
3773         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3774
3775             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3776                 qemu_devicedel($vmid, "cpu$i");
3777                 my $retry = 0;
3778                 my $currentrunningvcpus = undef;
3779                 while (1) {
3780                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3781                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3782                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3783                     $retry++;
3784                     sleep 1;
3785                 }
3786                 #update conf after each succesfull cpu unplug
3787                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3788                 PVE::QemuConfig->write_config($vmid, $conf);
3789             }
3790         } else {
3791             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3792         }
3793
3794         return;
3795     }
3796
3797     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3798     die "vcpus in running vm does not match its configuration\n"
3799         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3800
3801     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3802
3803         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3804             my $cpustr = print_cpu_device($conf, $i);
3805             qemu_deviceadd($vmid, $cpustr);
3806
3807             my $retry = 0;
3808             my $currentrunningvcpus = undef;
3809             while (1) {
3810                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3811                 last if scalar(@{$currentrunningvcpus}) == $i;
3812                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3813                 sleep 1;
3814                 $retry++;
3815             }
3816             #update conf after each succesfull cpu hotplug
3817             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3818             PVE::QemuConfig->write_config($vmid, $conf);
3819         }
3820     } else {
3821
3822         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3823             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3824         }
3825     }
3826 }
3827
3828 sub qemu_block_set_io_throttle {
3829     my ($vmid, $deviceid,
3830         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3831         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3832         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3833         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3834
3835     return if !check_running($vmid) ;
3836
3837     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3838         bps => int($bps),
3839         bps_rd => int($bps_rd),
3840         bps_wr => int($bps_wr),
3841         iops => int($iops),
3842         iops_rd => int($iops_rd),
3843         iops_wr => int($iops_wr),
3844         bps_max => int($bps_max),
3845         bps_rd_max => int($bps_rd_max),
3846         bps_wr_max => int($bps_wr_max),
3847         iops_max => int($iops_max),
3848         iops_rd_max => int($iops_rd_max),
3849         iops_wr_max => int($iops_wr_max),
3850         bps_max_length => int($bps_max_length),
3851         bps_rd_max_length => int($bps_rd_max_length),
3852         bps_wr_max_length => int($bps_wr_max_length),
3853         iops_max_length => int($iops_max_length),
3854         iops_rd_max_length => int($iops_rd_max_length),
3855         iops_wr_max_length => int($iops_wr_max_length),
3856     );
3857
3858 }
3859
3860 # old code, only used to shutdown old VM after update
3861 sub __read_avail {
3862     my ($fh, $timeout) = @_;
3863
3864     my $sel = new IO::Select;
3865     $sel->add($fh);
3866
3867     my $res = '';
3868     my $buf;
3869
3870     my @ready;
3871     while (scalar (@ready = $sel->can_read($timeout))) {
3872         my $count;
3873         if ($count = $fh->sysread($buf, 8192)) {
3874             if ($buf =~ /^(.*)\(qemu\) $/s) {
3875                 $res .= $1;
3876                 last;
3877             } else {
3878                 $res .= $buf;
3879             }
3880         } else {
3881             if (!defined($count)) {
3882                 die "$!\n";
3883             }
3884             last;
3885         }
3886     }
3887
3888     die "monitor read timeout\n" if !scalar(@ready);
3889
3890     return $res;
3891 }
3892
3893 # old code, only used to shutdown old VM after update
3894 sub vm_monitor_command {
3895     my ($vmid, $cmdstr, $nocheck) = @_;
3896
3897     my $res;
3898
3899     eval {
3900         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3901
3902         my $sname = "${var_run_tmpdir}/$vmid.mon";
3903
3904         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3905             die "unable to connect to VM $vmid socket - $!\n";
3906
3907         my $timeout = 3;
3908
3909         # hack: migrate sometime blocks the monitor (when migrate_downtime
3910         # is set)
3911         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3912             $timeout = 60*60; # 1 hour
3913         }
3914
3915         # read banner;
3916         my $data = __read_avail($sock, $timeout);
3917
3918         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3919             die "got unexpected qemu monitor banner\n";
3920         }
3921
3922         my $sel = new IO::Select;
3923         $sel->add($sock);
3924
3925         if (!scalar(my @ready = $sel->can_write($timeout))) {
3926             die "monitor write error - timeout";
3927         }
3928
3929         my $fullcmd = "$cmdstr\r";
3930
3931         # syslog('info', "VM $vmid monitor command: $cmdstr");
3932
3933         my $b;
3934         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3935             die "monitor write error - $!";
3936         }
3937
3938         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3939
3940         $timeout = 20;
3941
3942         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3943             $timeout = 60*60; # 1 hour
3944         } elsif ($cmdstr =~ m/^(eject|change)/) {
3945             $timeout = 60; # note: cdrom mount command is slow
3946         }
3947         if ($res = __read_avail($sock, $timeout)) {
3948
3949             my @lines = split("\r?\n", $res);
3950
3951             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3952
3953             $res = join("\n", @lines);
3954             $res .= "\n";
3955         }
3956     };
3957
3958     my $err = $@;
3959
3960     if ($err) {
3961         syslog("err", "VM $vmid monitor command failed - $err");
3962         die $err;
3963     }
3964
3965     return $res;
3966 }
3967
3968 sub qemu_block_resize {
3969     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3970
3971     my $running = check_running($vmid);
3972
3973     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3974
3975     return if !$running;
3976
3977     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3978
3979 }
3980
3981 sub qemu_volume_snapshot {
3982     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3983
3984     my $running = check_running($vmid);
3985
3986     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3987         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3988     } else {
3989         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3990     }
3991 }
3992
3993 sub qemu_volume_snapshot_delete {
3994     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3995
3996     my $running = check_running($vmid);
3997
3998     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3999         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4000     } else {
4001         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4002     }
4003 }
4004
4005 sub set_migration_caps {
4006     my ($vmid) = @_;
4007
4008     my $cap_ref = [];
4009
4010     my $enabled_cap = {
4011         "auto-converge" => 1,
4012         "xbzrle" => 1,
4013         "x-rdma-pin-all" => 0,
4014         "zero-blocks" => 0,
4015         "compress" => 0
4016     };
4017
4018     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4019
4020     for my $supported_capability (@$supported_capabilities) {
4021         push @$cap_ref, {
4022             capability => $supported_capability->{capability},
4023             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4024         };
4025     }
4026
4027     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4028 }
4029
4030 my $fast_plug_option = {
4031     'lock' => 1,
4032     'name' => 1,
4033     'onboot' => 1,
4034     'shares' => 1,
4035     'startup' => 1,
4036     'description' => 1,
4037     'protection' => 1,
4038 };
4039
4040 # hotplug changes in [PENDING]
4041 # $selection hash can be used to only apply specified options, for
4042 # example: { cores => 1 } (only apply changed 'cores')
4043 # $errors ref is used to return error messages
4044 sub vmconfig_hotplug_pending {
4045     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4046
4047     my $defaults = load_defaults();
4048
4049     # commit values which do not have any impact on running VM first
4050     # Note: those option cannot raise errors, we we do not care about
4051     # $selection and always apply them.
4052
4053     my $add_error = sub {
4054         my ($opt, $msg) = @_;
4055         $errors->{$opt} = "hotplug problem - $msg";
4056     };
4057
4058     my $changes = 0;
4059     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4060         if ($fast_plug_option->{$opt}) {
4061             $conf->{$opt} = $conf->{pending}->{$opt};
4062             delete $conf->{pending}->{$opt};
4063             $changes = 1;
4064         }
4065     }
4066
4067     if ($changes) {
4068         PVE::QemuConfig->write_config($vmid, $conf);
4069         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4070     }
4071
4072     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4073
4074     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4075     while (my ($opt, $force) = each %$pending_delete_hash) {
4076         next if $selection && !$selection->{$opt};
4077         eval {
4078             if ($opt eq 'hotplug') {
4079                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4080             } elsif ($opt eq 'tablet') {
4081                 die "skip\n" if !$hotplug_features->{usb};
4082                 if ($defaults->{tablet}) {
4083                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4084                 } else {
4085                     vm_deviceunplug($vmid, $conf, $opt);
4086                 }
4087             } elsif ($opt =~ m/^usb\d+/) {
4088                 die "skip\n";
4089                 # since we cannot reliably hot unplug usb devices
4090                 # we are disabling it
4091                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4092                 vm_deviceunplug($vmid, $conf, $opt);
4093             } elsif ($opt eq 'vcpus') {
4094                 die "skip\n" if !$hotplug_features->{cpu};
4095                 qemu_cpu_hotplug($vmid, $conf, undef);
4096             } elsif ($opt eq 'balloon') {
4097                 # enable balloon device is not hotpluggable
4098                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4099             } elsif ($fast_plug_option->{$opt}) {
4100                 # do nothing
4101             } elsif ($opt =~ m/^net(\d+)$/) {
4102                 die "skip\n" if !$hotplug_features->{network};
4103                 vm_deviceunplug($vmid, $conf, $opt);
4104             } elsif (is_valid_drivename($opt)) {
4105                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4106                 vm_deviceunplug($vmid, $conf, $opt);
4107                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4108             } elsif ($opt =~ m/^memory$/) {
4109                 die "skip\n" if !$hotplug_features->{memory};
4110                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4111             } elsif ($opt eq 'cpuunits') {
4112                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4113             } elsif ($opt eq 'cpulimit') {
4114                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4115             } else {
4116                 die "skip\n";
4117             }
4118         };
4119         if (my $err = $@) {
4120             &$add_error($opt, $err) if $err ne "skip\n";
4121         } else {
4122             # save new config if hotplug was successful
4123             delete $conf->{$opt};
4124             vmconfig_undelete_pending_option($conf, $opt);
4125             PVE::QemuConfig->write_config($vmid, $conf);
4126             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4127         }
4128     }
4129
4130     foreach my $opt (keys %{$conf->{pending}}) {
4131         next if $selection && !$selection->{$opt};
4132         my $value = $conf->{pending}->{$opt};
4133         eval {
4134             if ($opt eq 'hotplug') {
4135                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4136             } elsif ($opt eq 'tablet') {
4137                 die "skip\n" if !$hotplug_features->{usb};
4138                 if ($value == 1) {
4139                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4140                 } elsif ($value == 0) {
4141                     vm_deviceunplug($vmid, $conf, $opt);
4142                 }
4143             } elsif ($opt =~ m/^usb\d+$/) {
4144                 die "skip\n";
4145                 # since we cannot reliably hot unplug usb devices
4146                 # we are disabling it
4147                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4148                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4149                 die "skip\n" if !$d;
4150                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4151             } elsif ($opt eq 'vcpus') {
4152                 die "skip\n" if !$hotplug_features->{cpu};
4153                 qemu_cpu_hotplug($vmid, $conf, $value);
4154             } elsif ($opt eq 'balloon') {
4155                 # enable/disable balloning device is not hotpluggable
4156                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4157                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4158                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4159
4160                 # allow manual ballooning if shares is set to zero
4161                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4162                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4163                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4164                 }
4165             } elsif ($opt =~ m/^net(\d+)$/) {
4166                 # some changes can be done without hotplug
4167                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4168                                     $vmid, $opt, $value);
4169             } elsif (is_valid_drivename($opt)) {
4170                 # some changes can be done without hotplug
4171                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4172                                      $vmid, $opt, $value, 1);
4173             } elsif ($opt =~ m/^memory$/) { #dimms
4174                 die "skip\n" if !$hotplug_features->{memory};
4175                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4176             } elsif ($opt eq 'cpuunits') {
4177                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4178             } elsif ($opt eq 'cpulimit') {
4179                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4180                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4181             } else {
4182                 die "skip\n";  # skip non-hot-pluggable options
4183             }
4184         };
4185         if (my $err = $@) {
4186             &$add_error($opt, $err) if $err ne "skip\n";
4187         } else {
4188             # save new config if hotplug was successful
4189             $conf->{$opt} = $value;
4190             delete $conf->{pending}->{$opt};
4191             PVE::QemuConfig->write_config($vmid, $conf);
4192             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4193         }
4194     }
4195 }
4196
4197 sub try_deallocate_drive {
4198     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4199
4200     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4201         my $volid = $drive->{file};
4202         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4203             my $sid = PVE::Storage::parse_volume_id($volid);
4204             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4205
4206             # check if the disk is really unused
4207             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4208                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4209             PVE::Storage::vdisk_free($storecfg, $volid);
4210             return 1;
4211         } else {
4212             # If vm is not owner of this disk remove from config
4213             return 1;
4214         }
4215     }
4216
4217     return undef;
4218 }
4219
4220 sub vmconfig_delete_or_detach_drive {
4221     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4222
4223     my $drive = parse_drive($opt, $conf->{$opt});
4224
4225     my $rpcenv = PVE::RPCEnvironment::get();
4226     my $authuser = $rpcenv->get_user();
4227
4228     if ($force) {
4229         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4230         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4231     } else {
4232         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4233     }
4234 }
4235
4236 sub vmconfig_apply_pending {
4237     my ($vmid, $conf, $storecfg) = @_;
4238
4239     # cold plug
4240
4241     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4242     while (my ($opt, $force) = each %$pending_delete_hash) {
4243         die "internal error" if $opt =~ m/^unused/;
4244         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4245         if (!defined($conf->{$opt})) {
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             PVE::QemuConfig->write_config($vmid, $conf);
4248         } elsif (is_valid_drivename($opt)) {
4249             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4250             vmconfig_undelete_pending_option($conf, $opt);
4251             delete $conf->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253         } else {
4254             vmconfig_undelete_pending_option($conf, $opt);
4255             delete $conf->{$opt};
4256             PVE::QemuConfig->write_config($vmid, $conf);
4257         }
4258     }
4259
4260     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4261
4262     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4263         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4264
4265         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4266             # skip if nothing changed
4267         } elsif (is_valid_drivename($opt)) {
4268             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4269                 if defined($conf->{$opt});
4270             $conf->{$opt} = $conf->{pending}->{$opt};
4271         } else {
4272             $conf->{$opt} = $conf->{pending}->{$opt};
4273         }
4274
4275         delete $conf->{pending}->{$opt};
4276         PVE::QemuConfig->write_config($vmid, $conf);
4277     }
4278 }
4279
4280 my $safe_num_ne = sub {
4281     my ($a, $b) = @_;
4282
4283     return 0 if !defined($a) && !defined($b);
4284     return 1 if !defined($a);
4285     return 1 if !defined($b);
4286
4287     return $a != $b;
4288 };
4289
4290 my $safe_string_ne = sub {
4291     my ($a, $b) = @_;
4292
4293     return 0 if !defined($a) && !defined($b);
4294     return 1 if !defined($a);
4295     return 1 if !defined($b);
4296
4297     return $a ne $b;
4298 };
4299
4300 sub vmconfig_update_net {
4301     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4302
4303     my $newnet = parse_net($value);
4304
4305     if ($conf->{$opt}) {
4306         my $oldnet = parse_net($conf->{$opt});
4307
4308         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4309             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4310             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4311             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4312
4313             # for non online change, we try to hot-unplug
4314             die "skip\n" if !$hotplug;
4315             vm_deviceunplug($vmid, $conf, $opt);
4316         } else {
4317
4318             die "internal error" if $opt !~ m/net(\d+)/;
4319             my $iface = "tap${vmid}i$1";
4320
4321             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4322                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4323                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4324                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4325                 PVE::Network::tap_unplug($iface);
4326                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4327             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4328                 # Rate can be applied on its own but any change above needs to
4329                 # include the rate in tap_plug since OVS resets everything.
4330                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4331             }
4332
4333             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4334                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4335             }
4336
4337             return 1;
4338         }
4339     }
4340
4341     if ($hotplug) {
4342         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4343     } else {
4344         die "skip\n";
4345     }
4346 }
4347
4348 sub vmconfig_update_disk {
4349     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4350
4351     # fixme: do we need force?
4352
4353     my $drive = parse_drive($opt, $value);
4354
4355     if ($conf->{$opt}) {
4356
4357         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4358
4359             my $media = $drive->{media} || 'disk';
4360             my $oldmedia = $old_drive->{media} || 'disk';
4361             die "unable to change media type\n" if $media ne $oldmedia;
4362
4363             if (!drive_is_cdrom($old_drive)) {
4364
4365                 if ($drive->{file} ne $old_drive->{file}) {
4366
4367                     die "skip\n" if !$hotplug;
4368
4369                     # unplug and register as unused
4370                     vm_deviceunplug($vmid, $conf, $opt);
4371                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4372
4373                 } else {
4374                     # update existing disk
4375
4376                     # skip non hotpluggable value
4377                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4378                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4379                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4380                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4381                         die "skip\n";
4382                     }
4383
4384                     # apply throttle
4385                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4386                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4387                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4388                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4389                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4390                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4391                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4392                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4393                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4394                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4395                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4396                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4397                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4398                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4399                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4400                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4401                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4402                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4403
4404                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4405                                                    ($drive->{mbps} || 0)*1024*1024,
4406                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4407                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4408                                                    $drive->{iops} || 0,
4409                                                    $drive->{iops_rd} || 0,
4410                                                    $drive->{iops_wr} || 0,
4411                                                    ($drive->{mbps_max} || 0)*1024*1024,
4412                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4413                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4414                                                    $drive->{iops_max} || 0,
4415                                                    $drive->{iops_rd_max} || 0,
4416                                                    $drive->{iops_wr_max} || 0,
4417                                                    $drive->{bps_max_length} || 1,
4418                                                    $drive->{bps_rd_max_length} || 1,
4419                                                    $drive->{bps_wr_max_length} || 1,
4420                                                    $drive->{iops_max_length} || 1,
4421                                                    $drive->{iops_rd_max_length} || 1,
4422                                                    $drive->{iops_wr_max_length} || 1);
4423
4424                     }
4425
4426                     return 1;
4427                 }
4428
4429             } else { # cdrom
4430
4431                 if ($drive->{file} eq 'none') {
4432                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4433                 } else {
4434                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4435                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4436                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4437                 }
4438
4439                 return 1;
4440             }
4441         }
4442     }
4443
4444     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4445     # hotplug new disks
4446     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4447     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4448 }
4449
4450 sub vm_start {
4451     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4452         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4453
4454     PVE::QemuConfig->lock_config($vmid, sub {
4455         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4456
4457         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4458
4459         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4460
4461         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4462
4463         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4464             vmconfig_apply_pending($vmid, $conf, $storecfg);
4465             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4466         }
4467
4468         my $defaults = load_defaults();
4469
4470         # set environment variable useful inside network script
4471         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4472
4473         my $local_volumes = {};
4474
4475         if ($targetstorage) {
4476             foreach_drive($conf, sub {
4477                 my ($ds, $drive) = @_;
4478
4479                 return if drive_is_cdrom($drive);
4480
4481                 my $volid = $drive->{file};
4482
4483                 return if !$volid;
4484
4485                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4486
4487                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4488                 return if $scfg->{shared};
4489                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4490             });
4491
4492             my $format = undef;
4493
4494             foreach my $opt (sort keys %$local_volumes) {
4495
4496                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4497                 my $drive = parse_drive($opt, $conf->{$opt});
4498
4499                 #if remote storage is specified, use default format
4500                 if ($targetstorage && $targetstorage ne "1") {
4501                     $storeid = $targetstorage;
4502                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4503                     $format = $defFormat;
4504                 } else {
4505                     #else we use same format than original
4506                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4507                     $format = qemu_img_format($scfg, $volid);
4508                 }
4509
4510                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4511                 my $newdrive = $drive;
4512                 $newdrive->{format} = $format;
4513                 $newdrive->{file} = $newvolid;
4514                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4515                 $local_volumes->{$opt} = $drivestr;
4516                 #pass drive to conf for command line
4517                 $conf->{$opt} = $drivestr;
4518             }
4519         }
4520
4521         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4522
4523         my $migrate_port = 0;
4524         my $migrate_uri;
4525         if ($statefile) {
4526             if ($statefile eq 'tcp') {
4527                 my $localip = "localhost";
4528                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4529                 my $nodename = PVE::INotify::nodename();
4530
4531                 if (!defined($migration_type)) {
4532                     if (defined($datacenterconf->{migration}->{type})) {
4533                         $migration_type = $datacenterconf->{migration}->{type};
4534                     } else {
4535                         $migration_type = 'secure';
4536                     }
4537                 }
4538
4539                 if ($migration_type eq 'insecure') {
4540                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4541                     if ($migrate_network_addr) {
4542                         $localip = $migrate_network_addr;
4543                     } else {
4544                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4545                     }
4546
4547                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4548                 }
4549
4550                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4551                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4552                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4553                 push @$cmd, '-incoming', $migrate_uri;
4554                 push @$cmd, '-S';
4555
4556             } elsif ($statefile eq 'unix') {
4557                 # should be default for secure migrations as a ssh TCP forward
4558                 # tunnel is not deterministic reliable ready and fails regurarly
4559                 # to set up in time, so use UNIX socket forwards
4560                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4561                 unlink $socket_addr;
4562
4563                 $migrate_uri = "unix:$socket_addr";
4564
4565                 push @$cmd, '-incoming', $migrate_uri;
4566                 push @$cmd, '-S';
4567
4568             } else {
4569                 push @$cmd, '-loadstate', $statefile;
4570             }
4571         } elsif ($paused) {
4572             push @$cmd, '-S';
4573         }
4574
4575         # host pci devices
4576         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4577           my $d = parse_hostpci($conf->{"hostpci$i"});
4578           next if !$d;
4579           my $pcidevices = $d->{pciid};
4580           foreach my $pcidevice (@$pcidevices) {
4581                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4582
4583                 my $info = pci_device_info("0000:$pciid");
4584                 die "IOMMU not present\n" if !check_iommu_support();
4585                 die "no pci device info for device '$pciid'\n" if !$info;
4586                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4587                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4588           }
4589         }
4590
4591         PVE::Storage::activate_volumes($storecfg, $vollist);
4592
4593         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4594             my $cmd = [];
4595             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4596             eval  { run_command($cmd); };
4597         }
4598
4599         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4600                                                   : $defaults->{cpuunits};
4601
4602         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4603
4604         my %properties = (
4605             Slice => 'qemu.slice',
4606             KillMode => 'none',
4607             CPUShares => $cpuunits
4608         );
4609
4610         if (my $cpulimit = $conf->{cpulimit}) {
4611             $properties{CPUQuota} = int($cpulimit * 100);
4612         }
4613         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4614
4615         if ($conf->{hugepages}) {
4616
4617             my $code = sub {
4618                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4619                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4620
4621                 PVE::QemuServer::Memory::hugepages_mount();
4622                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4623
4624                 eval  {
4625                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4626                     run_command($cmd, %run_params);
4627                 };
4628
4629                 if (my $err = $@) {
4630                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4631                     die $err;
4632                 }
4633
4634                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4635             };
4636             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4637
4638         } else {
4639             eval  {
4640                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4641                 run_command($cmd, %run_params);
4642             };
4643         }
4644
4645         if (my $err = $@) {
4646             # deactivate volumes if start fails
4647             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4648             die "start failed: $err";
4649         }
4650
4651         print "migration listens on $migrate_uri\n" if $migrate_uri;
4652
4653         if ($statefile && $statefile ne 'tcp')  {
4654             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4655             warn $@ if $@;
4656         }
4657
4658         #start nbd server for storage migration
4659         if ($targetstorage) {
4660             my $nodename = PVE::INotify::nodename();
4661             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4662             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4663             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4664             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4665
4666             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4667
4668             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4669
4670             foreach my $opt (sort keys %$local_volumes) {
4671                 my $volid = $local_volumes->{$opt};
4672                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4673                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4674                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4675             }
4676         }
4677
4678         if ($migratedfrom) {
4679             eval {
4680                 set_migration_caps($vmid);
4681             };
4682             warn $@ if $@;
4683
4684             if ($spice_port) {
4685                 print "spice listens on port $spice_port\n";
4686                 if ($spice_ticket) {
4687                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4688                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4689                 }
4690             }
4691
4692         } else {
4693             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4694                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4695                     if $conf->{balloon};
4696             }
4697
4698             foreach my $opt (keys %$conf) {
4699                 next if $opt !~  m/^net\d+$/;
4700                 my $nicconf = parse_net($conf->{$opt});
4701                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4702             }
4703         }
4704
4705         vm_mon_cmd_nocheck($vmid, 'qom-set',
4706                     path => "machine/peripheral/balloon0",
4707                     property => "guest-stats-polling-interval",
4708                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4709
4710     });
4711 }
4712
4713 sub vm_mon_cmd {
4714     my ($vmid, $execute, %params) = @_;
4715
4716     my $cmd = { execute => $execute, arguments => \%params };
4717     vm_qmp_command($vmid, $cmd);
4718 }
4719
4720 sub vm_mon_cmd_nocheck {
4721     my ($vmid, $execute, %params) = @_;
4722
4723     my $cmd = { execute => $execute, arguments => \%params };
4724     vm_qmp_command($vmid, $cmd, 1);
4725 }
4726
4727 sub vm_qmp_command {
4728     my ($vmid, $cmd, $nocheck) = @_;
4729
4730     my $res;
4731
4732     my $timeout;
4733     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4734         $timeout = $cmd->{arguments}->{timeout};
4735         delete $cmd->{arguments}->{timeout};
4736     }
4737
4738     eval {
4739         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4740         my $sname = qmp_socket($vmid);
4741         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4742             my $qmpclient = PVE::QMPClient->new();
4743
4744             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4745         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4746             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4747                 if scalar(%{$cmd->{arguments}});
4748             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4749         } else {
4750             die "unable to open monitor socket\n";
4751         }
4752     };
4753     if (my $err = $@) {
4754         syslog("err", "VM $vmid qmp command failed - $err");
4755         die $err;
4756     }
4757
4758     return $res;
4759 }
4760
4761 sub vm_human_monitor_command {
4762     my ($vmid, $cmdline) = @_;
4763
4764     my $res;
4765
4766     my $cmd = {
4767         execute => 'human-monitor-command',
4768         arguments => { 'command-line' => $cmdline},
4769     };
4770
4771     return vm_qmp_command($vmid, $cmd);
4772 }
4773
4774 sub vm_commandline {
4775     my ($storecfg, $vmid) = @_;
4776
4777     my $conf = PVE::QemuConfig->load_config($vmid);
4778
4779     my $defaults = load_defaults();
4780
4781     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4782
4783     return PVE::Tools::cmd2string($cmd);
4784 }
4785
4786 sub vm_reset {
4787     my ($vmid, $skiplock) = @_;
4788
4789     PVE::QemuConfig->lock_config($vmid, sub {
4790
4791         my $conf = PVE::QemuConfig->load_config($vmid);
4792
4793         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4794
4795         vm_mon_cmd($vmid, "system_reset");
4796     });
4797 }
4798
4799 sub get_vm_volumes {
4800     my ($conf) = @_;
4801
4802     my $vollist = [];
4803     foreach_volid($conf, sub {
4804         my ($volid, $is_cdrom) = @_;
4805
4806         return if $volid =~ m|^/|;
4807
4808         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4809         return if !$sid;
4810
4811         push @$vollist, $volid;
4812     });
4813
4814     return $vollist;
4815 }
4816
4817 sub vm_stop_cleanup {
4818     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4819
4820     eval {
4821
4822         if (!$keepActive) {
4823             my $vollist = get_vm_volumes($conf);
4824             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4825         }
4826
4827         foreach my $ext (qw(mon qmp pid vnc qga)) {
4828             unlink "/var/run/qemu-server/${vmid}.$ext";
4829         }
4830
4831         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4832     };
4833     warn $@ if $@; # avoid errors - just warn
4834 }
4835
4836 # Note: use $nockeck to skip tests if VM configuration file exists.
4837 # We need that when migration VMs to other nodes (files already moved)
4838 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4839 sub vm_stop {
4840     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4841
4842     $force = 1 if !defined($force) && !$shutdown;
4843
4844     if ($migratedfrom){
4845         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4846         kill 15, $pid if $pid;
4847         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4848         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4849         return;
4850     }
4851
4852     PVE::QemuConfig->lock_config($vmid, sub {
4853
4854         my $pid = check_running($vmid, $nocheck);
4855         return if !$pid;
4856
4857         my $conf;
4858         if (!$nocheck) {
4859             $conf = PVE::QemuConfig->load_config($vmid);
4860             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4861             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4862                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4863                 $timeout = $opts->{down} if $opts->{down};
4864             }
4865         }
4866
4867         $timeout = 60 if !defined($timeout);
4868
4869         eval {
4870             if ($shutdown) {
4871                 if (defined($conf) && $conf->{agent}) {
4872                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4873                 } else {
4874                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4875                 }
4876             } else {
4877                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4878             }
4879         };
4880         my $err = $@;
4881
4882         if (!$err) {
4883             my $count = 0;
4884             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4885                 $count++;
4886                 sleep 1;
4887             }
4888
4889             if ($count >= $timeout) {
4890                 if ($force) {
4891                     warn "VM still running - terminating now with SIGTERM\n";
4892                     kill 15, $pid;
4893                 } else {
4894                     die "VM quit/powerdown failed - got timeout\n";
4895                 }
4896             } else {
4897                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4898                 return;
4899             }
4900         } else {
4901             if ($force) {
4902                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4903                 kill 15, $pid;
4904             } else {
4905                 die "VM quit/powerdown failed\n";
4906             }
4907         }
4908
4909         # wait again
4910         $timeout = 10;
4911
4912         my $count = 0;
4913         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4914             $count++;
4915             sleep 1;
4916         }
4917
4918         if ($count >= $timeout) {
4919             warn "VM still running - terminating now with SIGKILL\n";
4920             kill 9, $pid;
4921             sleep 1;
4922         }
4923
4924         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4925    });
4926 }
4927
4928 sub vm_suspend {
4929     my ($vmid, $skiplock) = @_;
4930
4931     PVE::QemuConfig->lock_config($vmid, sub {
4932
4933         my $conf = PVE::QemuConfig->load_config($vmid);
4934
4935         PVE::QemuConfig->check_lock($conf)
4936             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4937
4938         vm_mon_cmd($vmid, "stop");
4939     });
4940 }
4941
4942 sub vm_resume {
4943     my ($vmid, $skiplock, $nocheck) = @_;
4944
4945     PVE::QemuConfig->lock_config($vmid, sub {
4946
4947         if (!$nocheck) {
4948
4949             my $conf = PVE::QemuConfig->load_config($vmid);
4950
4951             PVE::QemuConfig->check_lock($conf)
4952                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4953
4954             vm_mon_cmd($vmid, "cont");
4955
4956         } else {
4957             vm_mon_cmd_nocheck($vmid, "cont");
4958         }
4959     });
4960 }
4961
4962 sub vm_sendkey {
4963     my ($vmid, $skiplock, $key) = @_;
4964
4965     PVE::QemuConfig->lock_config($vmid, sub {
4966
4967         my $conf = PVE::QemuConfig->load_config($vmid);
4968
4969         # there is no qmp command, so we use the human monitor command
4970         vm_human_monitor_command($vmid, "sendkey $key");
4971     });
4972 }
4973
4974 sub vm_destroy {
4975     my ($storecfg, $vmid, $skiplock) = @_;
4976
4977     PVE::QemuConfig->lock_config($vmid, sub {
4978
4979         my $conf = PVE::QemuConfig->load_config($vmid);
4980
4981         if (!check_running($vmid)) {
4982             destroy_vm($storecfg, $vmid, undef, $skiplock);
4983         } else {
4984             die "VM $vmid is running - destroy failed\n";
4985         }
4986     });
4987 }
4988
4989 # pci helpers
4990
4991 sub file_write {
4992     my ($filename, $buf) = @_;
4993
4994     my $fh = IO::File->new($filename, "w");
4995     return undef if !$fh;
4996
4997     my $res = print $fh $buf;
4998
4999     $fh->close();
5000
5001     return $res;
5002 }
5003
5004 sub pci_device_info {
5005     my ($name) = @_;
5006
5007     my $res;
5008
5009     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5010     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5011
5012     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5013     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5014
5015     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5016     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5017
5018     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5019     return undef if !defined($product) || $product !~ s/^0x//;
5020
5021     $res = {
5022         name => $name,
5023         vendor => $vendor,
5024         product => $product,
5025         domain => $domain,
5026         bus => $bus,
5027         slot => $slot,
5028         func => $func,
5029         irq => $irq,
5030         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5031     };
5032
5033     return $res;
5034 }
5035
5036 sub pci_dev_reset {
5037     my ($dev) = @_;
5038
5039     my $name = $dev->{name};
5040
5041     my $fn = "$pcisysfs/devices/$name/reset";
5042
5043     return file_write($fn, "1");
5044 }
5045
5046 sub pci_dev_bind_to_vfio {
5047     my ($dev) = @_;
5048
5049     my $name = $dev->{name};
5050
5051     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5052
5053     if (!-d $vfio_basedir) {
5054         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5055     }
5056     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5057
5058     my $testdir = "$vfio_basedir/$name";
5059     return 1 if -d $testdir;
5060
5061     my $data = "$dev->{vendor} $dev->{product}";
5062     return undef if !file_write("$vfio_basedir/new_id", $data);
5063
5064     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5065     if (!file_write($fn, $name)) {
5066         return undef if -f $fn;
5067     }
5068
5069     $fn = "$vfio_basedir/bind";
5070     if (! -d $testdir) {
5071         return undef if !file_write($fn, $name);
5072     }
5073
5074     return -d $testdir;
5075 }
5076
5077 sub pci_dev_group_bind_to_vfio {
5078     my ($pciid) = @_;
5079
5080     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5081
5082     if (!-d $vfio_basedir) {
5083         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5084     }
5085     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5086
5087     # get IOMMU group devices
5088     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5089       my @devs = grep /^0000:/, readdir($D);
5090     closedir($D);
5091
5092     foreach my $pciid (@devs) {
5093         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5094
5095         # pci bridges, switches or root ports are not supported
5096         # they have a pci_bus subdirectory so skip them
5097         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5098
5099         my $info = pci_device_info($1);
5100         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5101     }
5102
5103     return 1;
5104 }
5105
5106 # vzdump restore implementaion
5107
5108 sub tar_archive_read_firstfile {
5109     my $archive = shift;
5110
5111     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5112
5113     # try to detect archive type first
5114     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5115         die "unable to open file '$archive'\n";
5116     my $firstfile = <$fh>;
5117     kill 15, $pid;
5118     close $fh;
5119
5120     die "ERROR: archive contaions no data\n" if !$firstfile;
5121     chomp $firstfile;
5122
5123     return $firstfile;
5124 }
5125
5126 sub tar_restore_cleanup {
5127     my ($storecfg, $statfile) = @_;
5128
5129     print STDERR "starting cleanup\n";
5130
5131     if (my $fd = IO::File->new($statfile, "r")) {
5132         while (defined(my $line = <$fd>)) {
5133             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5134                 my $volid = $2;
5135                 eval {
5136                     if ($volid =~ m|^/|) {
5137                         unlink $volid || die 'unlink failed\n';
5138                     } else {
5139                         PVE::Storage::vdisk_free($storecfg, $volid);
5140                     }
5141                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5142                 };
5143                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5144             } else {
5145                 print STDERR "unable to parse line in statfile - $line";
5146             }
5147         }
5148         $fd->close();
5149     }
5150 }
5151
5152 sub restore_archive {
5153     my ($archive, $vmid, $user, $opts) = @_;
5154
5155     my $format = $opts->{format};
5156     my $comp;
5157
5158     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5159         $format = 'tar' if !$format;
5160         $comp = 'gzip';
5161     } elsif ($archive =~ m/\.tar$/) {
5162         $format = 'tar' if !$format;
5163     } elsif ($archive =~ m/.tar.lzo$/) {
5164         $format = 'tar' if !$format;
5165         $comp = 'lzop';
5166     } elsif ($archive =~ m/\.vma$/) {
5167         $format = 'vma' if !$format;
5168     } elsif ($archive =~ m/\.vma\.gz$/) {
5169         $format = 'vma' if !$format;
5170         $comp = 'gzip';
5171     } elsif ($archive =~ m/\.vma\.lzo$/) {
5172         $format = 'vma' if !$format;
5173         $comp = 'lzop';
5174     } else {
5175         $format = 'vma' if !$format; # default
5176     }
5177
5178     # try to detect archive format
5179     if ($format eq 'tar') {
5180         return restore_tar_archive($archive, $vmid, $user, $opts);
5181     } else {
5182         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5183     }
5184 }
5185
5186 sub restore_update_config_line {
5187     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5188
5189     return if $line =~ m/^\#qmdump\#/;
5190     return if $line =~ m/^\#vzdump\#/;
5191     return if $line =~ m/^lock:/;
5192     return if $line =~ m/^unused\d+:/;
5193     return if $line =~ m/^parent:/;
5194     return if $line =~ m/^template:/; # restored VM is never a template
5195
5196     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5197     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5198         # try to convert old 1.X settings
5199         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5200         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5201             my ($model, $macaddr) = split(/\=/, $devconfig);
5202             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5203             my $net = {
5204                 model => $model,
5205                 bridge => "vmbr$ind",
5206                 macaddr => $macaddr,
5207             };
5208             my $netstr = print_net($net);
5209
5210             print $outfd "net$cookie->{netcount}: $netstr\n";
5211             $cookie->{netcount}++;
5212         }
5213     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5214         my ($id, $netstr) = ($1, $2);
5215         my $net = parse_net($netstr);
5216         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5217         $netstr = print_net($net);
5218         print $outfd "$id: $netstr\n";
5219     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5220         my $virtdev = $1;
5221         my $value = $3;
5222         my $di = parse_drive($virtdev, $value);
5223         if (defined($di->{backup}) && !$di->{backup}) {
5224             print $outfd "#$line";
5225         } elsif ($map->{$virtdev}) {
5226             delete $di->{format}; # format can change on restore
5227             $di->{file} = $map->{$virtdev};
5228             $value = print_drive($vmid, $di);
5229             print $outfd "$virtdev: $value\n";
5230         } else {
5231             print $outfd $line;
5232         }
5233     } else {
5234         print $outfd $line;
5235     }
5236 }
5237
5238 sub scan_volids {
5239     my ($cfg, $vmid) = @_;
5240
5241     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5242
5243     my $volid_hash = {};
5244     foreach my $storeid (keys %$info) {
5245         foreach my $item (@{$info->{$storeid}}) {
5246             next if !($item->{volid} && $item->{size});
5247             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5248             $volid_hash->{$item->{volid}} = $item;
5249         }
5250     }
5251
5252     return $volid_hash;
5253 }
5254
5255 sub is_volume_in_use {
5256     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5257
5258     my $path = PVE::Storage::path($storecfg, $volid);
5259
5260     my $scan_config = sub {
5261         my ($cref, $snapname) = @_;
5262
5263         foreach my $key (keys %$cref) {
5264             my $value = $cref->{$key};
5265             if (is_valid_drivename($key)) {
5266                 next if $skip_drive && $key eq $skip_drive;
5267                 my $drive = parse_drive($key, $value);
5268                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5269                 return 1 if $volid eq $drive->{file};
5270                 if ($drive->{file} =~ m!^/!) {
5271                     return 1 if $drive->{file} eq $path;
5272                 } else {
5273                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5274                     next if !$storeid;
5275                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5276                     next if !$scfg;
5277                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5278                 }
5279             }
5280         }
5281
5282         return 0;
5283     };
5284
5285     return 1 if &$scan_config($conf);
5286
5287     undef $skip_drive;
5288
5289     foreach my $snapname (keys %{$conf->{snapshots}}) {
5290         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5291     }
5292
5293     return 0;
5294 }
5295
5296 sub update_disksize {
5297     my ($vmid, $conf, $volid_hash) = @_;
5298
5299     my $changes;
5300
5301     my $used = {};
5302
5303     # Note: it is allowed to define multiple storages with same path (alias), so
5304     # we need to check both 'volid' and real 'path' (two different volid can point
5305     # to the same path).
5306
5307     my $usedpath = {};
5308
5309     # update size info
5310     foreach my $opt (keys %$conf) {
5311         if (is_valid_drivename($opt)) {
5312             my $drive = parse_drive($opt, $conf->{$opt});
5313             my $volid = $drive->{file};
5314             next if !$volid;
5315
5316             $used->{$volid} = 1;
5317             if ($volid_hash->{$volid} &&
5318                 (my $path = $volid_hash->{$volid}->{path})) {
5319                 $usedpath->{$path} = 1;
5320             }
5321
5322             next if drive_is_cdrom($drive);
5323             next if !$volid_hash->{$volid};
5324
5325             $drive->{size} = $volid_hash->{$volid}->{size};
5326             my $new = print_drive($vmid, $drive);
5327             if ($new ne $conf->{$opt}) {
5328                 $changes = 1;
5329                 $conf->{$opt} = $new;
5330             }
5331         }
5332     }
5333
5334     # remove 'unusedX' entry if volume is used
5335     foreach my $opt (keys %$conf) {
5336         next if $opt !~ m/^unused\d+$/;
5337         my $volid = $conf->{$opt};
5338         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5339         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5340             $changes = 1;
5341             delete $conf->{$opt};
5342         }
5343     }
5344
5345     foreach my $volid (sort keys %$volid_hash) {
5346         next if $volid =~ m/vm-$vmid-state-/;
5347         next if $used->{$volid};
5348         my $path = $volid_hash->{$volid}->{path};
5349         next if !$path; # just to be sure
5350         next if $usedpath->{$path};
5351         $changes = 1;
5352         PVE::QemuConfig->add_unused_volume($conf, $volid);
5353         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5354     }
5355
5356     return $changes;
5357 }
5358
5359 sub rescan {
5360     my ($vmid, $nolock) = @_;
5361
5362     my $cfg = PVE::Storage::config();
5363
5364     my $volid_hash = scan_volids($cfg, $vmid);
5365
5366     my $updatefn =  sub {
5367         my ($vmid) = @_;
5368
5369         my $conf = PVE::QemuConfig->load_config($vmid);
5370
5371         PVE::QemuConfig->check_lock($conf);
5372
5373         my $vm_volids = {};
5374         foreach my $volid (keys %$volid_hash) {
5375             my $info = $volid_hash->{$volid};
5376             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5377         }
5378
5379         my $changes = update_disksize($vmid, $conf, $vm_volids);
5380
5381         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5382     };
5383
5384     if (defined($vmid)) {
5385         if ($nolock) {
5386             &$updatefn($vmid);
5387         } else {
5388             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5389         }
5390     } else {
5391         my $vmlist = config_list();
5392         foreach my $vmid (keys %$vmlist) {
5393             if ($nolock) {
5394                 &$updatefn($vmid);
5395             } else {
5396                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5397             }
5398         }
5399     }
5400 }
5401
5402 sub restore_vma_archive {
5403     my ($archive, $vmid, $user, $opts, $comp) = @_;
5404
5405     my $input = $archive eq '-' ? "<&STDIN" : undef;
5406     my $readfrom = $archive;
5407
5408     my $uncomp = '';
5409     if ($comp) {
5410         $readfrom = '-';
5411         my $qarchive = PVE::Tools::shellquote($archive);
5412         if ($comp eq 'gzip') {
5413             $uncomp = "zcat $qarchive|";
5414         } elsif ($comp eq 'lzop') {
5415             $uncomp = "lzop -d -c $qarchive|";
5416         } else {
5417             die "unknown compression method '$comp'\n";
5418         }
5419
5420     }
5421
5422     my $tmpdir = "/var/tmp/vzdumptmp$$";
5423     rmtree $tmpdir;
5424
5425     # disable interrupts (always do cleanups)
5426     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5427         warn "got interrupt - ignored\n";
5428     };
5429
5430     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5431     POSIX::mkfifo($mapfifo, 0600);
5432     my $fifofh;
5433
5434     my $openfifo = sub {
5435         open($fifofh, '>', $mapfifo) || die $!;
5436     };
5437
5438     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5439
5440     my $oldtimeout;
5441     my $timeout = 5;
5442
5443     my $devinfo = {};
5444
5445     my $rpcenv = PVE::RPCEnvironment::get();
5446
5447     my $conffile = PVE::QemuConfig->config_file($vmid);
5448     my $tmpfn = "$conffile.$$.tmp";
5449
5450     # Note: $oldconf is undef if VM does not exists
5451     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5452     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5453
5454     my $print_devmap = sub {
5455         my $virtdev_hash = {};
5456
5457         my $cfgfn = "$tmpdir/qemu-server.conf";
5458
5459         # we can read the config - that is already extracted
5460         my $fh = IO::File->new($cfgfn, "r") ||
5461             "unable to read qemu-server.conf - $!\n";
5462
5463         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5464         if (-f $fwcfgfn) {
5465             my $pve_firewall_dir = '/etc/pve/firewall';
5466             mkdir $pve_firewall_dir; # make sure the dir exists
5467             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5468         }
5469
5470         while (defined(my $line = <$fh>)) {
5471             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5472                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5473                 die "archive does not contain data for drive '$virtdev'\n"
5474                     if !$devinfo->{$devname};
5475                 if (defined($opts->{storage})) {
5476                     $storeid = $opts->{storage} || 'local';
5477                 } elsif (!$storeid) {
5478                     $storeid = 'local';
5479                 }
5480                 $format = 'raw' if !$format;
5481                 $devinfo->{$devname}->{devname} = $devname;
5482                 $devinfo->{$devname}->{virtdev} = $virtdev;
5483                 $devinfo->{$devname}->{format} = $format;
5484                 $devinfo->{$devname}->{storeid} = $storeid;
5485
5486                 # check permission on storage
5487                 my $pool = $opts->{pool}; # todo: do we need that?
5488                 if ($user ne 'root@pam') {
5489                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5490                 }
5491
5492                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5493             }
5494         }
5495
5496         foreach my $devname (keys %$devinfo) {
5497             die "found no device mapping information for device '$devname'\n"
5498                 if !$devinfo->{$devname}->{virtdev};
5499         }
5500
5501         my $cfg = PVE::Storage::config();
5502
5503         # create empty/temp config
5504         if ($oldconf) {
5505             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5506             foreach_drive($oldconf, sub {
5507                 my ($ds, $drive) = @_;
5508
5509                 return if drive_is_cdrom($drive);
5510
5511                 my $volid = $drive->{file};
5512
5513                 return if !$volid || $volid =~ m|^/|;
5514
5515                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5516                 return if !$path || !$owner || ($owner != $vmid);
5517
5518                 # Note: only delete disk we want to restore
5519                 # other volumes will become unused
5520                 if ($virtdev_hash->{$ds}) {
5521                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5522                     if (my $err = $@) {
5523                         warn $err;
5524                     }
5525                 }
5526             });
5527
5528             # delete vmstate files
5529             # since after the restore we have no snapshots anymore
5530             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5531                 my $snap = $oldconf->{snapshots}->{$snapname};
5532                 if ($snap->{vmstate}) {
5533                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5534                     if (my $err = $@) {
5535                         warn $err;
5536                     }
5537                 }
5538             }
5539         }
5540
5541         my $map = {};
5542         foreach my $virtdev (sort keys %$virtdev_hash) {
5543             my $d = $virtdev_hash->{$virtdev};
5544             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5545             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5546
5547             # test if requested format is supported
5548             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5549             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5550             $d->{format} = $defFormat if !$supported;
5551
5552             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5553                                                   $d->{format}, undef, $alloc_size);
5554             print STDERR "new volume ID is '$volid'\n";
5555             $d->{volid} = $volid;
5556             my $path = PVE::Storage::path($cfg, $volid);
5557
5558             PVE::Storage::activate_volumes($cfg,[$volid]);
5559
5560             my $write_zeros = 1;
5561             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5562                 $write_zeros = 0;
5563             }
5564
5565             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5566
5567             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5568             $map->{$virtdev} = $volid;
5569         }
5570
5571         $fh->seek(0, 0) || die "seek failed - $!\n";
5572
5573         my $outfd = new IO::File ($tmpfn, "w") ||
5574             die "unable to write config for VM $vmid\n";
5575
5576         my $cookie = { netcount => 0 };
5577         while (defined(my $line = <$fh>)) {
5578             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5579         }
5580
5581         $fh->close();
5582         $outfd->close();
5583     };
5584
5585     eval {
5586         # enable interrupts
5587         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5588             die "interrupted by signal\n";
5589         };
5590         local $SIG{ALRM} = sub { die "got timeout\n"; };
5591
5592         $oldtimeout = alarm($timeout);
5593
5594         my $parser = sub {
5595             my $line = shift;
5596
5597             print "$line\n";
5598
5599             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5600                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5601                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5602             } elsif ($line =~ m/^CTIME: /) {
5603                 # we correctly received the vma config, so we can disable
5604                 # the timeout now for disk allocation (set to 10 minutes, so
5605                 # that we always timeout if something goes wrong)
5606                 alarm(600);
5607                 &$print_devmap();
5608                 print $fifofh "done\n";
5609                 my $tmp = $oldtimeout || 0;
5610                 $oldtimeout = undef;
5611                 alarm($tmp);
5612                 close($fifofh);
5613             }
5614         };
5615
5616         print "restore vma archive: $cmd\n";
5617         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5618     };
5619     my $err = $@;
5620
5621     alarm($oldtimeout) if $oldtimeout;
5622
5623     my $vollist = [];
5624     foreach my $devname (keys %$devinfo) {
5625         my $volid = $devinfo->{$devname}->{volid};
5626         push @$vollist, $volid if $volid;
5627     }
5628
5629     my $cfg = PVE::Storage::config();
5630     PVE::Storage::deactivate_volumes($cfg, $vollist);
5631
5632     unlink $mapfifo;
5633
5634     if ($err) {
5635         rmtree $tmpdir;
5636         unlink $tmpfn;
5637
5638         foreach my $devname (keys %$devinfo) {
5639             my $volid = $devinfo->{$devname}->{volid};
5640             next if !$volid;
5641             eval {
5642                 if ($volid =~ m|^/|) {
5643                     unlink $volid || die 'unlink failed\n';
5644                 } else {
5645                     PVE::Storage::vdisk_free($cfg, $volid);
5646                 }
5647                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5648             };
5649             print STDERR "unable to cleanup '$volid' - $@" if $@;
5650         }
5651         die $err;
5652     }
5653
5654     rmtree $tmpdir;
5655
5656     rename($tmpfn, $conffile) ||
5657         die "unable to commit configuration file '$conffile'\n";
5658
5659     PVE::Cluster::cfs_update(); # make sure we read new file
5660
5661     eval { rescan($vmid, 1); };
5662     warn $@ if $@;
5663 }
5664
5665 sub restore_tar_archive {
5666     my ($archive, $vmid, $user, $opts) = @_;
5667
5668     if ($archive ne '-') {
5669         my $firstfile = tar_archive_read_firstfile($archive);
5670         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5671             if $firstfile ne 'qemu-server.conf';
5672     }
5673
5674     my $storecfg = PVE::Storage::config();
5675
5676     # destroy existing data - keep empty config
5677     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5678     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5679
5680     my $tocmd = "/usr/lib/qemu-server/qmextract";
5681
5682     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5683     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5684     $tocmd .= ' --prealloc' if $opts->{prealloc};
5685     $tocmd .= ' --info' if $opts->{info};
5686
5687     # tar option "xf" does not autodetect compression when read from STDIN,
5688     # so we pipe to zcat
5689     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5690         PVE::Tools::shellquote("--to-command=$tocmd");
5691
5692     my $tmpdir = "/var/tmp/vzdumptmp$$";
5693     mkpath $tmpdir;
5694
5695     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5696     local $ENV{VZDUMP_VMID} = $vmid;
5697     local $ENV{VZDUMP_USER} = $user;
5698
5699     my $conffile = PVE::QemuConfig->config_file($vmid);
5700     my $tmpfn = "$conffile.$$.tmp";
5701
5702     # disable interrupts (always do cleanups)
5703     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5704         print STDERR "got interrupt - ignored\n";
5705     };
5706
5707     eval {
5708         # enable interrupts
5709         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5710             die "interrupted by signal\n";
5711         };
5712
5713         if ($archive eq '-') {
5714             print "extracting archive from STDIN\n";
5715             run_command($cmd, input => "<&STDIN");
5716         } else {
5717             print "extracting archive '$archive'\n";
5718             run_command($cmd);
5719         }
5720
5721         return if $opts->{info};
5722
5723         # read new mapping
5724         my $map = {};
5725         my $statfile = "$tmpdir/qmrestore.stat";
5726         if (my $fd = IO::File->new($statfile, "r")) {
5727             while (defined (my $line = <$fd>)) {
5728                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5729                     $map->{$1} = $2 if $1;
5730                 } else {
5731                     print STDERR "unable to parse line in statfile - $line\n";
5732                 }
5733             }
5734             $fd->close();
5735         }
5736
5737         my $confsrc = "$tmpdir/qemu-server.conf";
5738
5739         my $srcfd = new IO::File($confsrc, "r") ||
5740             die "unable to open file '$confsrc'\n";
5741
5742         my $outfd = new IO::File ($tmpfn, "w") ||
5743             die "unable to write config for VM $vmid\n";
5744
5745         my $cookie = { netcount => 0 };
5746         while (defined (my $line = <$srcfd>)) {
5747             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5748         }
5749
5750         $srcfd->close();
5751         $outfd->close();
5752     };
5753     my $err = $@;
5754
5755     if ($err) {
5756
5757         unlink $tmpfn;
5758
5759         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5760
5761         die $err;
5762     }
5763
5764     rmtree $tmpdir;
5765
5766     rename $tmpfn, $conffile ||
5767         die "unable to commit configuration file '$conffile'\n";
5768
5769     PVE::Cluster::cfs_update(); # make sure we read new file
5770
5771     eval { rescan($vmid, 1); };
5772     warn $@ if $@;
5773 };
5774
5775 sub foreach_writable_storage {
5776     my ($conf, $func) = @_;
5777
5778     my $sidhash = {};
5779
5780     foreach my $ds (keys %$conf) {
5781         next if !is_valid_drivename($ds);
5782
5783         my $drive = parse_drive($ds, $conf->{$ds});
5784         next if !$drive;
5785         next if drive_is_cdrom($drive);
5786
5787         my $volid = $drive->{file};
5788
5789         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5790         $sidhash->{$sid} = $sid if $sid;
5791     }
5792
5793     foreach my $sid (sort keys %$sidhash) {
5794         &$func($sid);
5795     }
5796 }
5797
5798 sub do_snapshots_with_qemu {
5799     my ($storecfg, $volid) = @_;
5800
5801     my $storage_name = PVE::Storage::parse_volume_id($volid);
5802
5803     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5804         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5805         return 1;
5806     }
5807
5808     if ($volid =~ m/\.(qcow2|qed)$/){
5809         return 1;
5810     }
5811
5812     return undef;
5813 }
5814
5815 sub qga_check_running {
5816     my ($vmid) = @_;
5817
5818     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5819     if ($@) {
5820         warn "Qemu Guest Agent are not running - $@";
5821         return 0;
5822     }
5823     return 1;
5824 }
5825
5826 sub template_create {
5827     my ($vmid, $conf, $disk) = @_;
5828
5829     my $storecfg = PVE::Storage::config();
5830
5831     foreach_drive($conf, sub {
5832         my ($ds, $drive) = @_;
5833
5834         return if drive_is_cdrom($drive);
5835         return if $disk && $ds ne $disk;
5836
5837         my $volid = $drive->{file};
5838         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5839
5840         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5841         $drive->{file} = $voliddst;
5842         $conf->{$ds} = print_drive($vmid, $drive);
5843         PVE::QemuConfig->write_config($vmid, $conf);
5844     });
5845 }
5846
5847 sub qemu_img_convert {
5848     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5849
5850     my $storecfg = PVE::Storage::config();
5851     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5852     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5853
5854     if ($src_storeid && $dst_storeid) {
5855
5856         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5857
5858         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5859         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5860
5861         my $src_format = qemu_img_format($src_scfg, $src_volname);
5862         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5863
5864         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5865         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5866
5867         my $cmd = [];
5868         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5869         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5870         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5871         if ($is_zero_initialized) {
5872             push @$cmd, "zeroinit:$dst_path";
5873         } else {
5874             push @$cmd, $dst_path;
5875         }
5876
5877         my $parser = sub {
5878             my $line = shift;
5879             if($line =~ m/\((\S+)\/100\%\)/){
5880                 my $percent = $1;
5881                 my $transferred = int($size * $percent / 100);
5882                 my $remaining = $size - $transferred;
5883
5884                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5885             }
5886
5887         };
5888
5889         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5890         my $err = $@;
5891         die "copy failed: $err" if $err;
5892     }
5893 }
5894
5895 sub qemu_img_format {
5896     my ($scfg, $volname) = @_;
5897
5898     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5899         return $1;
5900     } else {
5901         return "raw";
5902     }
5903 }
5904
5905 sub qemu_drive_mirror {
5906     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5907
5908     $jobs = {} if !$jobs;
5909
5910     my $qemu_target;
5911     my $format;
5912     $jobs->{"drive-$drive"} = {};
5913
5914     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5915         my $server = $1;
5916         my $port = $2;
5917         my $exportname = $3;
5918
5919         $format = "nbd";
5920         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5921         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5922         my $cmd = ['socat', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=1"];
5923
5924         my $pid = fork();
5925         if (!defined($pid)) {
5926             die "forking socat tunnel failed";
5927         } elsif ($pid == 0) {
5928             exec(@$cmd);
5929             exit(-1);
5930         } else {
5931             $jobs->{"drive-$drive"}->{pid} = $pid;
5932
5933             my $timeout = 0;
5934             while (1) {
5935                 last if -S $unixsocket;
5936                 die if $timeout > 5;
5937                 $timeout++;
5938                 sleep 1;
5939             }
5940         }
5941     } else {
5942         my $storecfg = PVE::Storage::config();
5943         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5944
5945         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5946
5947         $format = qemu_img_format($dst_scfg, $dst_volname);
5948
5949         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5950
5951         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5952     }
5953
5954     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5955     $opts->{format} = $format if $format;
5956
5957     print "drive mirror is starting for drive-$drive\n";
5958
5959     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5960
5961     if (my $err = $@) {
5962         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5963         die "mirroring error: $err";
5964     }
5965
5966     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5967 }
5968
5969 sub qemu_drive_mirror_monitor {
5970     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5971
5972     eval {
5973         my $err_complete = 0;
5974
5975         while (1) {
5976             die "storage migration timed out\n" if $err_complete > 300;
5977
5978             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5979
5980             my $running_mirror_jobs = {};
5981             foreach my $stat (@$stats) {
5982                 next if $stat->{type} ne 'mirror';
5983                 $running_mirror_jobs->{$stat->{device}} = $stat;
5984             }
5985
5986             my $readycounter = 0;
5987
5988             foreach my $job (keys %$jobs) {
5989
5990                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
5991                     print "$job : finished\n";
5992                     delete $jobs->{$job};
5993                     next;
5994                 }
5995
5996                 die "$job: mirroring has been cancelled. Maybe do you have bad sectors?" if !defined($running_mirror_jobs->{$job});
5997
5998                 my $busy = $running_mirror_jobs->{$job}->{busy};
5999                 my $ready = $running_mirror_jobs->{$job}->{ready};
6000                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6001                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6002                     my $remaining = $total - $transferred;
6003                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6004
6005                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6006                 }
6007
6008                 $readycounter++ if $running_mirror_jobs->{$job}->{ready} eq 'true';
6009             }
6010
6011             last if scalar(keys %$jobs) == 0;
6012
6013             if ($readycounter == scalar(keys %$jobs)) {
6014                 print "all mirroring jobs are ready \n";
6015                 last if $skipcomplete; #do the complete later
6016
6017                 if ($vmiddst && $vmiddst != $vmid) {
6018                     if ($qga) {
6019                         print "freeze filesystem\n";
6020                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6021                     } else {
6022                         print "suspend vm\n";
6023                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6024                     }
6025
6026                     # if we clone a disk for a new target vm, we don't switch the disk
6027                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6028
6029                     if ($qga) {
6030                         print "unfreeze filesystem\n";
6031                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6032                     } else {
6033                         print "resume vm\n";
6034                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6035                     }
6036
6037                     last;
6038                 } else {
6039
6040                     foreach my $job (keys %$jobs) {
6041                         # try to switch the disk if source and destination are on the same guest
6042                         print "$job : Try to complete block job\n";
6043
6044                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6045                         if ($@ =~ m/cannot be completed/) {
6046                             print "$job : block job cannot be complete. Try again \n";
6047                             $err_complete++;
6048                         }else {
6049                             print "$job : complete ok : flushing pending writes\n";
6050                             $jobs->{$job}->{complete} = 1;
6051                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6052                         }
6053                     }
6054                 }
6055             }
6056             sleep 1;
6057         }
6058     };
6059     my $err = $@;
6060
6061     if ($err) {
6062         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6063         die "mirroring error: $err";
6064     }
6065
6066 }
6067
6068 sub qemu_blockjobs_cancel {
6069     my ($vmid, $jobs) = @_;
6070
6071     foreach my $job (keys %$jobs) {
6072         print "$job: try to cancel block job\n";
6073         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6074         $jobs->{$job}->{cancel} = 1;
6075     }
6076
6077     while (1) {
6078         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6079
6080         my $running_jobs = {};
6081         foreach my $stat (@$stats) {
6082             $running_jobs->{$stat->{device}} = $stat;
6083         }
6084
6085         foreach my $job (keys %$jobs) {
6086
6087             if(defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6088                 print "$job : finished\n";
6089                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6090                 delete $jobs->{$job};
6091             }
6092         }
6093
6094         last if scalar(keys %$jobs) == 0;
6095
6096         sleep 1;
6097     }
6098 }
6099
6100 sub qemu_blockjobs_finish_tunnel {
6101    my ($vmid, $job, $cpid) = @_;
6102
6103    return if !$cpid;
6104
6105    for (my $i = 1; $i < 20; $i++) {
6106         my $waitpid = waitpid($cpid, WNOHANG);
6107         last if (defined($waitpid) && ($waitpid == $cpid));
6108
6109         if ($i == 10) {
6110             kill(15, $cpid);
6111         } elsif ($i >= 15) {
6112             kill(9, $cpid);
6113         }
6114         sleep (1);
6115     }
6116     unlink "/run/qemu-server/$vmid.mirror-$job";
6117 }
6118
6119 sub clone_disk {
6120     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6121         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6122
6123     my $newvolid;
6124
6125     if (!$full) {
6126         print "create linked clone of drive $drivename ($drive->{file})\n";
6127         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6128         push @$newvollist, $newvolid;
6129     } else {
6130
6131         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6132         $storeid = $storage if $storage;
6133
6134         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6135         if (!$format) {
6136             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6137             $format = qemu_img_format($scfg, $volname);
6138         }
6139
6140         # test if requested format is supported - else use default
6141         my $supported = grep { $_ eq $format } @$validFormats;
6142         $format = $defFormat if !$supported;
6143
6144         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6145
6146         print "create full clone of drive $drivename ($drive->{file})\n";
6147         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6148         push @$newvollist, $newvolid;
6149
6150         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6151
6152         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6153         if (!$running || $snapname) {
6154             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6155         } else {
6156
6157             my $kvmver = get_running_qemu_version ($vmid);
6158             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6159                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6160                     if $drive->{iothread};
6161             }
6162
6163             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6164         }
6165     }
6166
6167     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6168
6169     my $disk = $drive;
6170     $disk->{format} = undef;
6171     $disk->{file} = $newvolid;
6172     $disk->{size} = $size;
6173
6174     return $disk;
6175 }
6176
6177 # this only works if VM is running
6178 sub get_current_qemu_machine {
6179     my ($vmid) = @_;
6180
6181     my $cmd = { execute => 'query-machines', arguments => {} };
6182     my $res = vm_qmp_command($vmid, $cmd);
6183
6184     my ($current, $default);
6185     foreach my $e (@$res) {
6186         $default = $e->{name} if $e->{'is-default'};
6187         $current = $e->{name} if $e->{'is-current'};
6188     }
6189
6190     # fallback to the default machine if current is not supported by qemu
6191     return $current || $default || 'pc';
6192 }
6193
6194 sub get_running_qemu_version {
6195     my ($vmid) = @_;
6196     my $cmd = { execute => 'query-version', arguments => {} };
6197     my $res = vm_qmp_command($vmid, $cmd);
6198     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6199 }
6200
6201 sub qemu_machine_feature_enabled {
6202     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6203
6204     my $current_major;
6205     my $current_minor;
6206
6207     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6208
6209         $current_major = $3;
6210         $current_minor = $4;
6211
6212     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6213
6214         $current_major = $1;
6215         $current_minor = $2;
6216     }
6217
6218     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6219
6220
6221 }
6222
6223 sub qemu_machine_pxe {
6224     my ($vmid, $conf, $machine) = @_;
6225
6226     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6227
6228     foreach my $opt (keys %$conf) {
6229         next if $opt !~ m/^net(\d+)$/;
6230         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6231         next if !$net;
6232         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6233         return $machine.".pxe" if $romfile =~ m/pxe/;
6234         last;
6235     }
6236
6237     return $machine;
6238 }
6239
6240 sub qemu_use_old_bios_files {
6241     my ($machine_type) = @_;
6242
6243     return if !$machine_type;
6244
6245     my $use_old_bios_files = undef;
6246
6247     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6248         $machine_type = $1;
6249         $use_old_bios_files = 1;
6250     } else {
6251         my $kvmver = kvm_user_version();
6252         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6253         # load new efi bios files on migration. So this hack is required to allow
6254         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6255         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6256         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6257     }
6258
6259     return ($use_old_bios_files, $machine_type);
6260 }
6261
6262 sub lspci {
6263
6264     my $devices = {};
6265
6266     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6267             my (undef, $id, $function) = @_;
6268             my $res = { id => $id, function => $function};
6269             push @{$devices->{$id}}, $res;
6270     });
6271
6272     return $devices;
6273 }
6274
6275 sub vm_iothreads_list {
6276     my ($vmid) = @_;
6277
6278     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6279
6280     my $iothreads = {};
6281     foreach my $iothread (@$res) {
6282         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6283     }
6284
6285     return $iothreads;
6286 }
6287
6288 sub scsihw_infos {
6289     my ($conf, $drive) = @_;
6290
6291     my $maxdev = 0;
6292
6293     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6294         $maxdev = 7;
6295     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6296         $maxdev = 1;
6297     } else {
6298         $maxdev = 256;
6299     }
6300
6301     my $controller = int($drive->{index} / $maxdev);
6302     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6303
6304     return ($maxdev, $controller, $controller_prefix);
6305 }
6306
6307 sub add_hyperv_enlighments {
6308     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6309
6310     return if $nokvm;
6311     return if $winversion < 6;
6312     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6313
6314     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6315
6316     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6317         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6318         push @$cpuFlags , 'hv_vapic';
6319         push @$cpuFlags , 'hv_time';
6320     } else {
6321         push @$cpuFlags , 'hv_spinlocks=0xffff';
6322     }
6323
6324     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6325         push @$cpuFlags , 'hv_reset';
6326         push @$cpuFlags , 'hv_vpindex';
6327         push @$cpuFlags , 'hv_runtime';
6328     }
6329
6330     if ($winversion >= 7) {
6331         push @$cpuFlags , 'hv_relaxed';
6332     }
6333 }
6334
6335 sub windows_version {
6336     my ($ostype) = @_;
6337
6338     return 0 if !$ostype;
6339
6340     my $winversion = 0;
6341
6342     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6343         $winversion = 5;
6344     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6345         $winversion = 6;
6346     } elsif ($ostype =~ m/^win(\d+)$/) {
6347         $winversion = $1;
6348     }
6349
6350     return $winversion;
6351 }
6352
6353 # bash completion helper
6354
6355 sub complete_backup_archives {
6356     my ($cmdname, $pname, $cvalue) = @_;
6357
6358     my $cfg = PVE::Storage::config();
6359
6360     my $storeid;
6361
6362     if ($cvalue =~ m/^([^:]+):/) {
6363         $storeid = $1;
6364     }
6365
6366     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6367
6368     my $res = [];
6369     foreach my $id (keys %$data) {
6370         foreach my $item (@{$data->{$id}}) {
6371             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6372             push @$res, $item->{volid} if defined($item->{volid});
6373         }
6374     }
6375
6376     return $res;
6377 }
6378
6379 my $complete_vmid_full = sub {
6380     my ($running) = @_;
6381
6382     my $idlist = vmstatus();
6383
6384     my $res = [];
6385
6386     foreach my $id (keys %$idlist) {
6387         my $d = $idlist->{$id};
6388         if (defined($running)) {
6389             next if $d->{template};
6390             next if $running && $d->{status} ne 'running';
6391             next if !$running && $d->{status} eq 'running';
6392         }
6393         push @$res, $id;
6394
6395     }
6396     return $res;
6397 };
6398
6399 sub complete_vmid {
6400     return &$complete_vmid_full();
6401 }
6402
6403 sub complete_vmid_stopped {
6404     return &$complete_vmid_full(0);
6405 }
6406
6407 sub complete_vmid_running {
6408     return &$complete_vmid_full(1);
6409 }
6410
6411 sub complete_storage {
6412
6413     my $cfg = PVE::Storage::config();
6414     my $ids = $cfg->{ids};
6415
6416     my $res = [];
6417     foreach my $sid (keys %$ids) {
6418         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6419         next if !$ids->{$sid}->{content}->{images};
6420         push @$res, $sid;
6421     }
6422
6423     return $res;
6424 }
6425
6426 sub nbd_stop {
6427     my ($vmid) = @_;
6428
6429     vm_mon_cmd($vmid, 'nbd-server-stop');
6430 }
6431
6432 1;