PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     rerror => {
 751         type => 'string',
 752         enum => [qw(ignore report stop)],
 753         description => 'Read error action.',
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %iothread_fmt = ( iothread => {
 790         type => 'boolean',
 791         description => "Whether to use iothreads for this drive",
 792         optional => 1,
 793 });
 794
 795 my %model_fmt = (
 796     model => {
 797         type => 'string',
 798         format => 'urlencoded',
 799         format_description => 'model',
 800         maxLength => 40*3, # *3 since it's %xx url enoded
 801         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 802         optional => 1,
 803     },
 804 );
 805
 806 my %queues_fmt = (
 807     queues => {
 808         type => 'integer',
 809         description => "Number of queues.",
 810         minimum => 2,
 811         optional => 1
 812     }
 813 );
 814
 815 my %scsiblock_fmt = (
 816     scsiblock => {
 817         type => 'boolean',
 818         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 819         optional => 1,
 820         default => 0,
 821     },
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 826     my $d = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what in $longunit.",
 830         optional => 1,
 831     };
 832     $d->{minimum} = $minimum if defined($minimum);
 833     $drivedesc_base{$key} = $d;
 834 };
 835 # throughput: (leaky bucket)
 836 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 838 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 839 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 841 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 842 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 843 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 844 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 845
 846 # pools: (pool of IO before throttling starts taking effect)
 847 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 850 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 851 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 852 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 853
 854 # burst lengths
 855 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 857 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 860 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 861
 862 my $ide_fmt = {
 863     %drivedesc_base,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879     %scsiblock_fmt,
 880 };
 881 my $scsidesc = {
 882     optional => 1,
 883     type => 'string', format => $scsi_fmt,
 884     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 887
 888 my $sata_fmt = {
 889     %drivedesc_base,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901 };
 902 my $virtiodesc = {
 903     optional => 1,
 904     type => 'string', format => $virtio_fmt,
 905     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 906 };
 907 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 908
 909 my $alldrive_fmt = {
 910     %drivedesc_base,
 911     %iothread_fmt,
 912     %model_fmt,
 913     %queues_fmt,
 914     %scsiblock_fmt,
 915 };
 916
 917 my $efidisk_fmt = {
 918     volume => { alias => 'file' },
 919     file => {
 920         type => 'string',
 921         format => 'pve-volume-id-or-qm-path',
 922         default_key => 1,
 923         format_description => 'volume',
 924         description => "The drive's backing volume.",
 925     },
 926     format => get_standard_option('pve-qm-image-format'),
 927     size => {
 928         type => 'string',
 929         format => 'disk-size',
 930         format_description => 'DiskSize',
 931         description => "Disk size. This is purely informational and has no effect.",
 932         optional => 1,
 933     },
 934 };
 935
 936 my $efidisk_desc = {
 937     optional => 1,
 938     type => 'string', format => $efidisk_fmt,
 939     description => "Configure a Disk for storing EFI vars",
 940 };
 941
 942 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 943
 944 my $usb_fmt = {
 945     host => {
 946         default_key => 1,
 947         type => 'string', format => 'pve-qm-usb-device',
 948         format_description => 'HOSTUSBDEVICE|spice',
 949         description => <<EODESCR,
 950 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 951
 952  'bus-port(.port)*' (decimal numbers) or
 953  'vendor_id:product_id' (hexadeciaml numbers) or
 954  'spice'
 955
 956 You can use the 'lsusb -t' command to list existing usb devices.
 957
 958 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 959
 960 The value 'spice' can be used to add a usb redirection devices for spice.
 961 EODESCR
 962     },
 963     usb3 => {
 964         optional => 1,
 965         type => 'boolean',
 966         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 967         default => 0,
 968     },
 969 };
 970
 971 my $usbdesc = {
 972     optional => 1,
 973     type => 'string', format => $usb_fmt,
 974     description => "Configure an USB device (n is 0 to 4).",
 975 };
 976 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 977
 978 # NOTE: the match-groups of this regex are used in parse_hostpci
 979 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 980 my $hostpci_fmt = {
 981     host => {
 982         default_key => 1,
 983         type => 'string',
 984         pattern => qr/$PCIRE(;$PCIRE)*/,
 985         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 986         description => <<EODESCR,
 987 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 988 of PCI virtual functions of the host. HOSTPCIID syntax is:
 989
 990 'bus:dev.func' (hexadecimal numbers)
 991
 992 You can us the 'lspci' command to list existing PCI devices.
 993 EODESCR
 994     },
 995     rombar => {
 996         type => 'boolean',
 997         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 998         optional => 1,
 999         default => 1,
1000     },
1001     romfile => {
1002         type => 'string',
1003         pattern => '[^,;]+',
1004         format_description => 'string',
1005         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1006         optional => 1,
1007     },
1008     pcie => {
1009         type => 'boolean',
1010         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1011         optional => 1,
1012         default => 0,
1013     },
1014     'x-vga' => {
1015         type => 'boolean',
1016         description =>  "Enable vfio-vga device support.",
1017         optional => 1,
1018         default => 0,
1019     },
1020 };
1021 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1022
1023 my $hostpcidesc = {
1024         optional => 1,
1025         type => 'string', format => 'pve-qm-hostpci',
1026         description => "Map host PCI devices into guest.",
1027         verbose_description =>  <<EODESCR,
1028 Map host PCI devices into guest.
1029
1030 NOTE: This option allows direct access to host hardware. So it is no longer
1031 possible to migrate such machines - use with special care.
1032
1033 CAUTION: Experimental! User reported problems with this option.
1034 EODESCR
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 my $paralleldesc= {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1058         description =>  "Map host parallel devices (n is 0 to 2).",
1059         verbose_description =>  <<EODESCR,
1060 Map host parallel devices (n is 0 to 2).
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $hostpcidesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1081     $drivename_hash->{"ide$i"} = 1;
1082     $confdesc->{"ide$i"} = $idedesc;
1083 }
1084
1085 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1086     $drivename_hash->{"sata$i"} = 1;
1087     $confdesc->{"sata$i"} = $satadesc;
1088 }
1089
1090 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1091     $drivename_hash->{"scsi$i"} = 1;
1092     $confdesc->{"scsi$i"} = $scsidesc ;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1096     $drivename_hash->{"virtio$i"} = 1;
1097     $confdesc->{"virtio$i"} = $virtiodesc;
1098 }
1099
1100 $drivename_hash->{efidisk0} = 1;
1101 $confdesc->{efidisk0} = $efidisk_desc;
1102
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1104     $confdesc->{"usb$i"} = $usbdesc;
1105 }
1106
1107 my $unuseddesc = {
1108     optional => 1,
1109     type => 'string', format => 'pve-volume-id',
1110     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1111 };
1112
1113 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1114     $confdesc->{"unused$i"} = $unuseddesc;
1115 }
1116
1117 my $kvm_api_version = 0;
1118
1119 sub kvm_version {
1120
1121     return $kvm_api_version if $kvm_api_version;
1122
1123     my $fh = IO::File->new("</dev/kvm") ||
1124         return 0;
1125
1126     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1127         $kvm_api_version = $v;
1128     }
1129
1130     $fh->close();
1131
1132     return  $kvm_api_version;
1133 }
1134
1135 my $kvm_user_version;
1136
1137 sub kvm_user_version {
1138
1139     return $kvm_user_version if $kvm_user_version;
1140
1141     $kvm_user_version = 'unknown';
1142
1143     my $code = sub {
1144         my $line = shift;
1145         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1146             $kvm_user_version = $2;
1147         }
1148     };
1149
1150     eval { run_command("kvm -version", outfunc => $code); };
1151     warn $@ if $@;
1152
1153     return $kvm_user_version;
1154
1155 }
1156
1157 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1158
1159 sub valid_drive_names {
1160     # order is important - used to autoselect boot disk
1161     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1162             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1163             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1164             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1165             'efidisk0');
1166 }
1167
1168 sub is_valid_drivename {
1169     my $dev = shift;
1170
1171     return defined($drivename_hash->{$dev});
1172 }
1173
1174 sub option_exists {
1175     my $key = shift;
1176     return defined($confdesc->{$key});
1177 }
1178
1179 sub nic_models {
1180     return $nic_model_list;
1181 }
1182
1183 sub os_list_description {
1184
1185     return {
1186         other => 'Other',
1187         wxp => 'Windows XP',
1188         w2k => 'Windows 2000',
1189         w2k3 =>, 'Windows 2003',
1190         w2k8 => 'Windows 2008',
1191         wvista => 'Windows Vista',
1192         win7 => 'Windows 7',
1193         win8 => 'Windows 8/2012',
1194         win10 => 'Windows 10/2016',
1195         l24 => 'Linux 2.4',
1196         l26 => 'Linux 2.6',
1197     };
1198 }
1199
1200 my $cdrom_path;
1201
1202 sub get_cdrom_path {
1203
1204     return  $cdrom_path if $cdrom_path;
1205
1206     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1207     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1208     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1209 }
1210
1211 sub get_iso_path {
1212     my ($storecfg, $vmid, $cdrom) = @_;
1213
1214     if ($cdrom eq 'cdrom') {
1215         return get_cdrom_path();
1216     } elsif ($cdrom eq 'none') {
1217         return '';
1218     } elsif ($cdrom =~ m|^/|) {
1219         return $cdrom;
1220     } else {
1221         return PVE::Storage::path($storecfg, $cdrom);
1222     }
1223 }
1224
1225 # try to convert old style file names to volume IDs
1226 sub filename_to_volume_id {
1227     my ($vmid, $file, $media) = @_;
1228
1229     if (!($file eq 'none' || $file eq 'cdrom' ||
1230           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1231
1232         return undef if $file =~ m|/|;
1233
1234         if ($media && $media eq 'cdrom') {
1235             $file = "local:iso/$file";
1236         } else {
1237             $file = "local:$vmid/$file";
1238         }
1239     }
1240
1241     return $file;
1242 }
1243
1244 sub verify_media_type {
1245     my ($opt, $vtype, $media) = @_;
1246
1247     return if !$media;
1248
1249     my $etype;
1250     if ($media eq 'disk') {
1251         $etype = 'images';
1252     } elsif ($media eq 'cdrom') {
1253         $etype = 'iso';
1254     } else {
1255         die "internal error";
1256     }
1257
1258     return if ($vtype eq $etype);
1259
1260     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1261 }
1262
1263 sub cleanup_drive_path {
1264     my ($opt, $storecfg, $drive) = @_;
1265
1266     # try to convert filesystem paths to volume IDs
1267
1268     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1269         ($drive->{file} !~ m|^/dev/.+|) &&
1270         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1271         ($drive->{file} !~ m/^\d+$/)) {
1272         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1273         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1274         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1275         verify_media_type($opt, $vtype, $drive->{media});
1276         $drive->{file} = $volid;
1277     }
1278
1279     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1280 }
1281
1282 sub parse_hotplug_features {
1283     my ($data) = @_;
1284
1285     my $res = {};
1286
1287     return $res if $data eq '0';
1288
1289     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1290
1291     foreach my $feature (PVE::Tools::split_list($data)) {
1292         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1293             $res->{$1} = 1;
1294         } else {
1295             die "invalid hotplug feature '$feature'\n";
1296         }
1297     }
1298     return $res;
1299 }
1300
1301 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1302 sub pve_verify_hotplug_features {
1303     my ($value, $noerr) = @_;
1304
1305     return $value if parse_hotplug_features($value);
1306
1307     return undef if $noerr;
1308
1309     die "unable to parse hotplug option\n";
1310 }
1311
1312 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1313 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1314 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1315 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1316 #        [,iothread=on][,serial=serial][,model=model]
1317
1318 sub parse_drive {
1319     my ($key, $data) = @_;
1320
1321     my ($interface, $index);
1322
1323     if ($key =~ m/^([^\d]+)(\d+)$/) {
1324         $interface = $1;
1325         $index = $2;
1326     } else {
1327         return undef;
1328     }
1329
1330     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1331                                      : $confdesc->{$key}->{format};
1332     if (!$desc) {
1333         warn "invalid drive key: $key\n";
1334         return undef;
1335     }
1336     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1337     return undef if !$res;
1338     $res->{interface} = $interface;
1339     $res->{index} = $index;
1340
1341     my $error = 0;
1342     foreach my $opt (qw(bps bps_rd bps_wr)) {
1343         if (my $bps = defined(delete $res->{$opt})) {
1344             if (defined($res->{"m$opt"})) {
1345                 warn "both $opt and m$opt specified\n";
1346                 ++$error;
1347                 next;
1348             }
1349             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1350         }
1351     }
1352
1353     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1354     for my $requirement (
1355         [bps_max_length => 'mbps_max'],
1356         [bps_rd_max_length => 'mbps_rd_max'],
1357         [bps_wr_max_length => 'mbps_wr_max'],
1358         [iops_max_length => 'iops_max'],
1359         [iops_rd_max_length => 'iops_rd_max'],
1360         [iops_wr_max_length => 'iops_wr_max']) {
1361         my ($option, $requires) = @$requirement;
1362         if ($res->{$option} && !$res->{$requires}) {
1363             warn "$option requires $requires\n";
1364             ++$error;
1365         }
1366     }
1367
1368     return undef if $error;
1369
1370     return undef if $res->{mbps_rd} && $res->{mbps};
1371     return undef if $res->{mbps_wr} && $res->{mbps};
1372     return undef if $res->{iops_rd} && $res->{iops};
1373     return undef if $res->{iops_wr} && $res->{iops};
1374
1375     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1376         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1377         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1378         return undef if $res->{interface} eq 'virtio';
1379     }
1380
1381     if (my $size = $res->{size}) {
1382         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1383     }
1384
1385     return $res;
1386 }
1387
1388 sub print_drive {
1389     my ($vmid, $drive) = @_;
1390     my $data = { %$drive };
1391     delete $data->{$_} for qw(index interface);
1392     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1393 }
1394
1395 sub scsi_inquiry {
1396     my($fh, $noerr) = @_;
1397
1398     my $SG_IO = 0x2285;
1399     my $SG_GET_VERSION_NUM = 0x2282;
1400
1401     my $versionbuf = "\x00" x 8;
1402     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1403     if (!$ret) {
1404         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1405         return undef;
1406     }
1407     my $version = unpack("I", $versionbuf);
1408     if ($version < 30000) {
1409         die "scsi generic interface too old\n"  if !$noerr;
1410         return undef;
1411     }
1412
1413     my $buf = "\x00" x 36;
1414     my $sensebuf = "\x00" x 8;
1415     my $cmd = pack("C x3 C x1", 0x12, 36);
1416
1417     # see /usr/include/scsi/sg.h
1418     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1419
1420     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1421                       length($sensebuf), 0, length($buf), $buf,
1422                       $cmd, $sensebuf, 6000);
1423
1424     $ret = ioctl($fh, $SG_IO, $packet);
1425     if (!$ret) {
1426         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my @res = unpack($sg_io_hdr_t, $packet);
1431     if ($res[17] || $res[18]) {
1432         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1433         return undef;
1434     }
1435
1436     my $res = {};
1437     (my $byte0, my $byte1, $res->{vendor},
1438      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1439
1440     $res->{removable} = $byte1 & 128 ? 1 : 0;
1441     $res->{type} = $byte0 & 31;
1442
1443     return $res;
1444 }
1445
1446 sub path_is_scsi {
1447     my ($path) = @_;
1448
1449     my $fh = IO::File->new("+<$path") || return undef;
1450     my $res = scsi_inquiry($fh, 1);
1451     close($fh);
1452
1453     return $res;
1454 }
1455
1456 sub machine_type_is_q35 {
1457     my ($conf) = @_;
1458
1459     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf) = @_;
1464
1465     my $q35 = machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus = $q35 ? "ehci" : "uhci";
1469
1470     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1471 }
1472
1473 sub print_drivedevice_full {
1474     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1475
1476     my $device = '';
1477     my $maxdev = 0;
1478
1479     if ($drive->{interface} eq 'virtio') {
1480         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1481         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1482         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1483     } elsif ($drive->{interface} eq 'scsi') {
1484
1485         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1486         my $unit = $drive->{index} % $maxdev;
1487         my $devicetype = 'hd';
1488         my $path = '';
1489         if (drive_is_cdrom($drive)) {
1490             $devicetype = 'cd';
1491         } else {
1492             if ($drive->{file} =~ m|^/|) {
1493                 $path = $drive->{file};
1494                 if (my $info = path_is_scsi($path)) {
1495                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1496                         $devicetype = 'block';
1497                     } elsif ($info->{type} == 1) { # tape
1498                         $devicetype = 'generic';
1499                     }
1500                 }
1501             } else {
1502                  $path = PVE::Storage::path($storecfg, $drive->{file});
1503             }
1504
1505             if($path =~ m/^iscsi\:\/\//){
1506                 $devicetype = 'generic';
1507             }
1508         }
1509
1510         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1512         } else {
1513             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1514         }
1515
1516     } elsif ($drive->{interface} eq 'ide'){
1517         $maxdev = 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1523         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1524             $model = URI::Escape::uri_unescape($model);
1525             $device .= ",model=$model";
1526         }
1527     } elsif ($drive->{interface} eq 'sata'){
1528         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1529         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1530         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1531     } elsif ($drive->{interface} eq 'usb') {
1532         die "implement me";
1533         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1534     } else {
1535         die "unsupported interface type";
1536     }
1537
1538     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1539
1540     return $device;
1541 }
1542
1543 sub get_initiator_name {
1544     my $initiator;
1545
1546     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1547     while (defined(my $line = <$fh>)) {
1548         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1549         $initiator = $1;
1550         last;
1551     }
1552     $fh->close();
1553
1554     return $initiator;
1555 }
1556
1557 sub print_drive_full {
1558     my ($storecfg, $vmid, $drive) = @_;
1559
1560     my $path;
1561     my $volid = $drive->{file};
1562     my $format;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566     } else {
1567         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1571             $format = qemu_img_format($scfg, $volname);
1572         } else {
1573             $path = $volid;
1574             $format = "raw";
1575         }
1576    }
1577
1578     my $opts = '';
1579     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1580     foreach my $o (@qemu_drive_options) {
1581         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1582     }
1583     if (my $serial = $drive->{serial}) {
1584         $serial = URI::Escape::uri_unescape($serial);
1585         $opts .= ",serial=$serial";
1586     }
1587
1588     $opts .= ",format=$format" if $format && !$drive->{format};
1589
1590     foreach my $o (qw(bps bps_rd bps_wr)) {
1591         my $v = $drive->{"m$o"};
1592         $opts .= ",$o=" . int($v*1024*1024) if $v;
1593     }
1594
1595     my $cache_direct = 0;
1596
1597     if (my $cache = $drive->{cache}) {
1598         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1599     } elsif (!drive_is_cdrom($drive)) {
1600         $opts .= ",cache=none";
1601         $cache_direct = 1;
1602     }
1603
1604     # aio native works only with O_DIRECT
1605     if (!$drive->{aio}) {
1606         if($cache_direct) {
1607             $opts .= ",aio=native";
1608         } else {
1609             $opts .= ",aio=threads";
1610         }
1611     }
1612
1613     if (!drive_is_cdrom($drive)) {
1614         my $detectzeroes;
1615         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1616             $detectzeroes = 'off';
1617         } elsif ($drive->{discard}) {
1618             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1619         } else {
1620             # This used to be our default with discard not being specified:
1621             $detectzeroes = 'on';
1622         }
1623         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     my $pathinfo = $path ? "file=$path," : '';
1627
1628     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1629 }
1630
1631 sub print_netdevice_full {
1632     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1633
1634     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1635
1636     my $device = $net->{model};
1637     if ($net->{model} eq 'virtio') {
1638          $device = 'virtio-net-pci';
1639      };
1640
1641     my $pciaddr = print_pci_addr("$netid", $bridges);
1642     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1643     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1644         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1645         my $vectors = $net->{queues} * 2 + 2;
1646         $tmpstr .= ",vectors=$vectors,mq=on";
1647     }
1648     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1649
1650     if ($use_old_bios_files) {
1651         my $romfile;
1652         if ($device eq 'virtio-net-pci') {
1653             $romfile = 'pxe-virtio.rom';
1654         } elsif ($device eq 'e1000') {
1655             $romfile = 'pxe-e1000.rom';
1656         } elsif ($device eq 'ne2k') {
1657             $romfile = 'pxe-ne2k_pci.rom';
1658         } elsif ($device eq 'pcnet') {
1659             $romfile = 'pxe-pcnet.rom';
1660         } elsif ($device eq 'rtl8139') {
1661             $romfile = 'pxe-rtl8139.rom';
1662         }
1663         $tmpstr .= ",romfile=$romfile" if $romfile;
1664     }
1665
1666     return $tmpstr;
1667 }
1668
1669 sub print_netdev_full {
1670     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1671
1672     my $i = '';
1673     if ($netid =~ m/^net(\d+)$/) {
1674         $i = int($1);
1675     }
1676
1677     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1678
1679     my $ifname = "tap${vmid}i$i";
1680
1681     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1682     die "interface name '$ifname' is too long (max 15 character)\n"
1683         if length($ifname) >= 16;
1684
1685     my $vhostparam = '';
1686     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1687
1688     my $vmname = $conf->{name} || "vm$vmid";
1689
1690     my $netdev = "";
1691     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1692
1693     if ($net->{bridge}) {
1694         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695     } else {
1696         $netdev = "type=user,id=$netid,hostname=$vmname";
1697     }
1698
1699     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701     return $netdev;
1702 }
1703
1704
1705 sub print_cpu_device {
1706     my ($conf, $id) = @_;
1707
1708     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1709     my $cpu = $nokvm ? "qemu64" : "kvm64";
1710     if (my $cputype = $conf->{cpu}) {
1711         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1712             or die "Cannot parse cpu description: $cputype\n";
1713         $cpu = $cpuconf->{cputype};
1714     }
1715
1716     my $sockets = 1;
1717     $sockets = $conf->{sockets} if  $conf->{sockets};
1718     my $cores = $conf->{cores} || 1;
1719
1720     my $current_core = ($id - 1) % $cores;
1721     my $current_socket = int(($id - $current_core)/$cores);
1722
1723     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1724 }
1725
1726 sub drive_is_cdrom {
1727     my ($drive) = @_;
1728
1729     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1730
1731 }
1732
1733 sub parse_number_sets {
1734     my ($set) = @_;
1735     my $res = [];
1736     foreach my $part (split(/;/, $set)) {
1737         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1738             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1739             push @$res, [ $1, $2 ];
1740         } else {
1741             die "invalid range: $part\n";
1742         }
1743     }
1744     return $res;
1745 }
1746
1747 sub parse_numa {
1748     my ($data) = @_;
1749
1750     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1751     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1752     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1753     return $res;
1754 }
1755
1756 sub parse_hostpci {
1757     my ($value) = @_;
1758
1759     return undef if !$value;
1760
1761     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1762
1763     my @idlist = split(/;/, $res->{host});
1764     delete $res->{host};
1765     foreach my $id (@idlist) {
1766         if ($id =~ /^$PCIRE$/) {
1767             if (defined($2)) {
1768                 push @{$res->{pciid}}, { id => $1, function => $2 };
1769             } else {
1770                 my $pcidevices = lspci($1);
1771                 $res->{pciid} = $pcidevices->{$1};
1772             }
1773         } else {
1774             # should have been caught by parse_property_string already
1775             die "failed to parse PCI id: $id\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1782 sub parse_net {
1783     my ($data) = @_;
1784
1785     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1786     if ($@) {
1787         warn $@;
1788         return undef;
1789     }
1790     if (!defined($res->{macaddr})) {
1791         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1792         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1793     }
1794     return $res;
1795 }
1796
1797 sub print_net {
1798     my $net = shift;
1799
1800     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1801 }
1802
1803 sub add_random_macs {
1804     my ($settings) = @_;
1805
1806     foreach my $opt (keys %$settings) {
1807         next if $opt !~ m/^net(\d+)$/;
1808         my $net = parse_net($settings->{$opt});
1809         next if !$net;
1810         $settings->{$opt} = print_net($net);
1811     }
1812 }
1813
1814 sub vm_is_volid_owner {
1815     my ($storecfg, $vmid, $volid) = @_;
1816
1817     if ($volid !~  m|^/|) {
1818         my ($path, $owner);
1819         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1820         if ($owner && ($owner == $vmid)) {
1821             return 1;
1822         }
1823     }
1824
1825     return undef;
1826 }
1827
1828 sub split_flagged_list {
1829     my $text = shift || '';
1830     $text =~ s/[,;]/ /g;
1831     $text =~ s/^\s+//;
1832     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1833 }
1834
1835 sub join_flagged_list {
1836     my ($how, $lst) = @_;
1837     join $how, map { $lst->{$_} . $_ } keys %$lst;
1838 }
1839
1840 sub vmconfig_delete_pending_option {
1841     my ($conf, $key, $force) = @_;
1842
1843     delete $conf->{pending}->{$key};
1844     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1845     $pending_delete_hash->{$key} = $force ? '!' : '';
1846     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1847 }
1848
1849 sub vmconfig_undelete_pending_option {
1850     my ($conf, $key) = @_;
1851
1852     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1853     delete $pending_delete_hash->{$key};
1854
1855     if (%$pending_delete_hash) {
1856         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1857     } else {
1858         delete $conf->{pending}->{delete};
1859     }
1860 }
1861
1862 sub vmconfig_register_unused_drive {
1863     my ($storecfg, $vmid, $conf, $drive) = @_;
1864
1865     if (!drive_is_cdrom($drive)) {
1866         my $volid = $drive->{file};
1867         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1868             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1869         }
1870     }
1871 }
1872
1873 sub vmconfig_cleanup_pending {
1874     my ($conf) = @_;
1875
1876     # remove pending changes when nothing changed
1877     my $changes;
1878     foreach my $opt (keys %{$conf->{pending}}) {
1879         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1880             $changes = 1;
1881             delete $conf->{pending}->{$opt};
1882         }
1883     }
1884
1885     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1886     my $pending_delete_hash = {};
1887     while (my ($opt, $force) = each %$current_delete_hash) {
1888         if (defined($conf->{$opt})) {
1889             $pending_delete_hash->{$opt} = $force;
1890         } else {
1891             $changes = 1;
1892         }
1893     }
1894
1895     if (%$pending_delete_hash) {
1896         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1897     } else {
1898         delete $conf->{pending}->{delete};
1899     }
1900
1901     return $changes;
1902 }
1903
1904 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1905 my $smbios1_fmt = {
1906     uuid => {
1907         type => 'string',
1908         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1909         format_description => 'UUID',
1910         description => "Set SMBIOS1 UUID.",
1911         optional => 1,
1912     },
1913     version => {
1914         type => 'string',
1915         pattern => '\S+',
1916         format_description => 'string',
1917         description => "Set SMBIOS1 version.",
1918         optional => 1,
1919     },
1920     serial => {
1921         type => 'string',
1922         pattern => '\S+',
1923         format_description => 'string',
1924         description => "Set SMBIOS1 serial number.",
1925         optional => 1,
1926     },
1927     manufacturer => {
1928         type => 'string',
1929         pattern => '\S+',
1930         format_description => 'string',
1931         description => "Set SMBIOS1 manufacturer.",
1932         optional => 1,
1933     },
1934     product => {
1935         type => 'string',
1936         pattern => '\S+',
1937         format_description => 'string',
1938         description => "Set SMBIOS1 product ID.",
1939         optional => 1,
1940     },
1941     sku => {
1942         type => 'string',
1943         pattern => '\S+',
1944         format_description => 'string',
1945         description => "Set SMBIOS1 SKU string.",
1946         optional => 1,
1947     },
1948     family => {
1949         type => 'string',
1950         pattern => '\S+',
1951         format_description => 'string',
1952         description => "Set SMBIOS1 family string.",
1953         optional => 1,
1954     },
1955 };
1956
1957 sub parse_smbios1 {
1958     my ($data) = @_;
1959
1960     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1961     warn $@ if $@;
1962     return $res;
1963 }
1964
1965 sub print_smbios1 {
1966     my ($smbios1) = @_;
1967     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1968 }
1969
1970 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1971
1972 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1973 sub verify_bootdisk {
1974     my ($value, $noerr) = @_;
1975
1976     return $value if is_valid_drivename($value);
1977
1978     return undef if $noerr;
1979
1980     die "invalid boot disk '$value'\n";
1981 }
1982
1983 sub parse_watchdog {
1984     my ($value) = @_;
1985
1986     return undef if !$value;
1987
1988     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1989     warn $@ if $@;
1990     return $res;
1991 }
1992
1993 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1994 sub verify_usb_device {
1995     my ($value, $noerr) = @_;
1996
1997     return $value if parse_usb_device($value);
1998
1999     return undef if $noerr;
2000
2001     die "unable to parse usb device\n";
2002 }
2003
2004 # add JSON properties for create and set function
2005 sub json_config_properties {
2006     my $prop = shift;
2007
2008     foreach my $opt (keys %$confdesc) {
2009         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2010         $prop->{$opt} = $confdesc->{$opt};
2011     }
2012
2013     return $prop;
2014 }
2015
2016 sub check_type {
2017     my ($key, $value) = @_;
2018
2019     die "unknown setting '$key'\n" if !$confdesc->{$key};
2020
2021     my $type = $confdesc->{$key}->{type};
2022
2023     if (!defined($value)) {
2024         die "got undefined value\n";
2025     }
2026
2027     if ($value =~ m/[\n\r]/) {
2028         die "property contains a line feed\n";
2029     }
2030
2031     if ($type eq 'boolean') {
2032         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2033         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2034         die "type check ('boolean') failed - got '$value'\n";
2035     } elsif ($type eq 'integer') {
2036         return int($1) if $value =~ m/^(\d+)$/;
2037         die "type check ('integer') failed - got '$value'\n";
2038     } elsif ($type eq 'number') {
2039         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2040         die "type check ('number') failed - got '$value'\n";
2041     } elsif ($type eq 'string') {
2042         if (my $fmt = $confdesc->{$key}->{format}) {
2043             PVE::JSONSchema::check_format($fmt, $value);
2044             return $value;
2045         }
2046         $value =~ s/^\"(.*)\"$/$1/;
2047         return $value;
2048     } else {
2049         die "internal error"
2050     }
2051 }
2052
2053 sub check_iommu_support{
2054     #fixme : need to check IOMMU support
2055     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2056
2057     my $iommu=1;
2058     return $iommu;
2059
2060 }
2061
2062 sub touch_config {
2063     my ($vmid) = @_;
2064
2065     my $conf = PVE::QemuConfig->config_file($vmid);
2066     utime undef, undef, $conf;
2067 }
2068
2069 sub destroy_vm {
2070     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2071
2072     my $conffile = PVE::QemuConfig->config_file($vmid);
2073
2074     my $conf = PVE::QemuConfig->load_config($vmid);
2075
2076     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2077
2078     # only remove disks owned by this VM
2079     foreach_drive($conf, sub {
2080         my ($ds, $drive) = @_;
2081
2082         return if drive_is_cdrom($drive);
2083
2084         my $volid = $drive->{file};
2085
2086         return if !$volid || $volid =~ m|^/|;
2087
2088         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2089         return if !$path || !$owner || ($owner != $vmid);
2090
2091         eval {
2092             PVE::Storage::vdisk_free($storecfg, $volid);
2093         };
2094         warn "Could not remove disk '$volid', check manually: $@" if $@;
2095
2096     });
2097
2098     if ($keep_empty_config) {
2099         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2100     } else {
2101         unlink $conffile;
2102     }
2103
2104     # also remove unused disk
2105     eval {
2106         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2107
2108         eval {
2109             PVE::Storage::foreach_volid($dl, sub {
2110                 my ($volid, $sid, $volname, $d) = @_;
2111                 PVE::Storage::vdisk_free($storecfg, $volid);
2112             });
2113         };
2114         warn $@ if $@;
2115
2116     };
2117     warn $@ if $@;
2118 }
2119
2120 sub parse_vm_config {
2121     my ($filename, $raw) = @_;
2122
2123     return undef if !defined($raw);
2124
2125     my $res = {
2126         digest => Digest::SHA::sha1_hex($raw),
2127         snapshots => {},
2128         pending => {},
2129     };
2130
2131     $filename =~ m|/qemu-server/(\d+)\.conf$|
2132         || die "got strange filename '$filename'";
2133
2134     my $vmid = $1;
2135
2136     my $conf = $res;
2137     my $descr;
2138     my $section = '';
2139
2140     my @lines = split(/\n/, $raw);
2141     foreach my $line (@lines) {
2142         next if $line =~ m/^\s*$/;
2143
2144         if ($line =~ m/^\[PENDING\]\s*$/i) {
2145             $section = 'pending';
2146             if (defined($descr)) {
2147                 $descr =~ s/\s+$//;
2148                 $conf->{description} = $descr;
2149             }
2150             $descr = undef;
2151             $conf = $res->{$section} = {};
2152             next;
2153
2154         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2155             $section = $1;
2156             if (defined($descr)) {
2157                 $descr =~ s/\s+$//;
2158                 $conf->{description} = $descr;
2159             }
2160             $descr = undef;
2161             $conf = $res->{snapshots}->{$section} = {};
2162             next;
2163         }
2164
2165         if ($line =~ m/^\#(.*)\s*$/) {
2166             $descr = '' if !defined($descr);
2167             $descr .= PVE::Tools::decode_text($1) . "\n";
2168             next;
2169         }
2170
2171         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2172             $descr = '' if !defined($descr);
2173             $descr .= PVE::Tools::decode_text($2);
2174         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2175             $conf->{snapstate} = $1;
2176         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2177             my $key = $1;
2178             my $value = $2;
2179             $conf->{$key} = $value;
2180         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2181             my $value = $1;
2182             if ($section eq 'pending') {
2183                 $conf->{delete} = $value; # we parse this later
2184             } else {
2185                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2186             }
2187         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2188             my $key = $1;
2189             my $value = $2;
2190             eval { $value = check_type($key, $value); };
2191             if ($@) {
2192                 warn "vm $vmid - unable to parse value of '$key' - $@";
2193             } else {
2194                 $key = 'ide2' if $key eq 'cdrom';
2195                 my $fmt = $confdesc->{$key}->{format};
2196                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2197                     my $v = parse_drive($key, $value);
2198                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2199                         $v->{file} = $volid;
2200                         $value = print_drive($vmid, $v);
2201                     } else {
2202                         warn "vm $vmid - unable to parse value of '$key'\n";
2203                         next;
2204                     }
2205                 }
2206
2207                 $conf->{$key} = $value;
2208             }
2209         }
2210     }
2211
2212     if (defined($descr)) {
2213         $descr =~ s/\s+$//;
2214         $conf->{description} = $descr;
2215     }
2216     delete $res->{snapstate}; # just to be sure
2217
2218     return $res;
2219 }
2220
2221 sub write_vm_config {
2222     my ($filename, $conf) = @_;
2223
2224     delete $conf->{snapstate}; # just to be sure
2225
2226     if ($conf->{cdrom}) {
2227         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2228         $conf->{ide2} = $conf->{cdrom};
2229         delete $conf->{cdrom};
2230     }
2231
2232     # we do not use 'smp' any longer
2233     if ($conf->{sockets}) {
2234         delete $conf->{smp};
2235     } elsif ($conf->{smp}) {
2236         $conf->{sockets} = $conf->{smp};
2237         delete $conf->{cores};
2238         delete $conf->{smp};
2239     }
2240
2241     my $used_volids = {};
2242
2243     my $cleanup_config = sub {
2244         my ($cref, $pending, $snapname) = @_;
2245
2246         foreach my $key (keys %$cref) {
2247             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2248                 $key eq 'snapstate' || $key eq 'pending';
2249             my $value = $cref->{$key};
2250             if ($key eq 'delete') {
2251                 die "propertry 'delete' is only allowed in [PENDING]\n"
2252                     if !$pending;
2253                 # fixme: check syntax?
2254                 next;
2255             }
2256             eval { $value = check_type($key, $value); };
2257             die "unable to parse value of '$key' - $@" if $@;
2258
2259             $cref->{$key} = $value;
2260
2261             if (!$snapname && is_valid_drivename($key)) {
2262                 my $drive = parse_drive($key, $value);
2263                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2264             }
2265         }
2266     };
2267
2268     &$cleanup_config($conf);
2269
2270     &$cleanup_config($conf->{pending}, 1);
2271
2272     foreach my $snapname (keys %{$conf->{snapshots}}) {
2273         die "internal error" if $snapname eq 'pending';
2274         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2275     }
2276
2277     # remove 'unusedX' settings if we re-add a volume
2278     foreach my $key (keys %$conf) {
2279         my $value = $conf->{$key};
2280         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2281             delete $conf->{$key};
2282         }
2283     }
2284
2285     my $generate_raw_config = sub {
2286         my ($conf, $pending) = @_;
2287
2288         my $raw = '';
2289
2290         # add description as comment to top of file
2291         if (defined(my $descr = $conf->{description})) {
2292             if ($descr) {
2293                 foreach my $cl (split(/\n/, $descr)) {
2294                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2295                 }
2296             } else {
2297                 $raw .= "#\n" if $pending;
2298             }
2299         }
2300
2301         foreach my $key (sort keys %$conf) {
2302             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2303             $raw .= "$key: $conf->{$key}\n";
2304         }
2305         return $raw;
2306     };
2307
2308     my $raw = &$generate_raw_config($conf);
2309
2310     if (scalar(keys %{$conf->{pending}})){
2311         $raw .= "\n[PENDING]\n";
2312         $raw .= &$generate_raw_config($conf->{pending}, 1);
2313     }
2314
2315     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2316         $raw .= "\n[$snapname]\n";
2317         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2318     }
2319
2320     return $raw;
2321 }
2322
2323 sub load_defaults {
2324
2325     my $res = {};
2326
2327     # we use static defaults from our JSON schema configuration
2328     foreach my $key (keys %$confdesc) {
2329         if (defined(my $default = $confdesc->{$key}->{default})) {
2330             $res->{$key} = $default;
2331         }
2332     }
2333
2334     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2335     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2336
2337     return $res;
2338 }
2339
2340 sub config_list {
2341     my $vmlist = PVE::Cluster::get_vmlist();
2342     my $res = {};
2343     return $res if !$vmlist || !$vmlist->{ids};
2344     my $ids = $vmlist->{ids};
2345
2346     foreach my $vmid (keys %$ids) {
2347         my $d = $ids->{$vmid};
2348         next if !$d->{node} || $d->{node} ne $nodename;
2349         next if !$d->{type} || $d->{type} ne 'qemu';
2350         $res->{$vmid}->{exists} = 1;
2351     }
2352     return $res;
2353 }
2354
2355 # test if VM uses local resources (to prevent migration)
2356 sub check_local_resources {
2357     my ($conf, $noerr) = @_;
2358
2359     my $loc_res = 0;
2360
2361     $loc_res = 1 if $conf->{hostusb}; # old syntax
2362     $loc_res = 1 if $conf->{hostpci}; # old syntax
2363
2364     foreach my $k (keys %$conf) {
2365         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2366         # sockets are safe: they will recreated be on the target side post-migrate
2367         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2368         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2369     }
2370
2371     die "VM uses local resources\n" if $loc_res && !$noerr;
2372
2373     return $loc_res;
2374 }
2375
2376 # check if used storages are available on all nodes (use by migrate)
2377 sub check_storage_availability {
2378     my ($storecfg, $conf, $node) = @_;
2379
2380     foreach_drive($conf, sub {
2381         my ($ds, $drive) = @_;
2382
2383         my $volid = $drive->{file};
2384         return if !$volid;
2385
2386         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2387         return if !$sid;
2388
2389         # check if storage is available on both nodes
2390         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2391         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2392    });
2393 }
2394
2395 # list nodes where all VM images are available (used by has_feature API)
2396 sub shared_nodes {
2397     my ($conf, $storecfg) = @_;
2398
2399     my $nodelist = PVE::Cluster::get_nodelist();
2400     my $nodehash = { map { $_ => 1 } @$nodelist };
2401     my $nodename = PVE::INotify::nodename();
2402
2403     foreach_drive($conf, sub {
2404         my ($ds, $drive) = @_;
2405
2406         my $volid = $drive->{file};
2407         return if !$volid;
2408
2409         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2410         if ($storeid) {
2411             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2412             if ($scfg->{disable}) {
2413                 $nodehash = {};
2414             } elsif (my $avail = $scfg->{nodes}) {
2415                 foreach my $node (keys %$nodehash) {
2416                     delete $nodehash->{$node} if !$avail->{$node};
2417                 }
2418             } elsif (!$scfg->{shared}) {
2419                 foreach my $node (keys %$nodehash) {
2420                     delete $nodehash->{$node} if $node ne $nodename
2421                 }
2422             }
2423         }
2424     });
2425
2426     return $nodehash
2427 }
2428
2429 sub check_cmdline {
2430     my ($pidfile, $pid) = @_;
2431
2432     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2433     if (defined($fh)) {
2434         my $line = <$fh>;
2435         $fh->close;
2436         return undef if !$line;
2437         my @param = split(/\0/, $line);
2438
2439         my $cmd = $param[0];
2440         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2441
2442         for (my $i = 0; $i < scalar (@param); $i++) {
2443             my $p = $param[$i];
2444             next if !$p;
2445             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2446                 my $p = $param[$i+1];
2447                 return 1 if $p && ($p eq $pidfile);
2448                 return undef;
2449             }
2450         }
2451     }
2452     return undef;
2453 }
2454
2455 sub check_running {
2456     my ($vmid, $nocheck, $node) = @_;
2457
2458     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2459
2460     die "unable to find configuration file for VM $vmid - no such machine\n"
2461         if !$nocheck && ! -f $filename;
2462
2463     my $pidfile = pidfile_name($vmid);
2464
2465     if (my $fd = IO::File->new("<$pidfile")) {
2466         my $st = stat($fd);
2467         my $line = <$fd>;
2468         close($fd);
2469
2470         my $mtime = $st->mtime;
2471         if ($mtime > time()) {
2472             warn "file '$filename' modified in future\n";
2473         }
2474
2475         if ($line =~ m/^(\d+)$/) {
2476             my $pid = $1;
2477             if (check_cmdline($pidfile, $pid)) {
2478                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2479                     return $pid;
2480                 }
2481             }
2482         }
2483     }
2484
2485     return undef;
2486 }
2487
2488 sub vzlist {
2489
2490     my $vzlist = config_list();
2491
2492     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2493
2494     while (defined(my $de = $fd->read)) {
2495         next if $de !~ m/^(\d+)\.pid$/;
2496         my $vmid = $1;
2497         next if !defined($vzlist->{$vmid});
2498         if (my $pid = check_running($vmid)) {
2499             $vzlist->{$vmid}->{pid} = $pid;
2500         }
2501     }
2502
2503     return $vzlist;
2504 }
2505
2506 sub disksize {
2507     my ($storecfg, $conf) = @_;
2508
2509     my $bootdisk = $conf->{bootdisk};
2510     return undef if !$bootdisk;
2511     return undef if !is_valid_drivename($bootdisk);
2512
2513     return undef if !$conf->{$bootdisk};
2514
2515     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2516     return undef if !defined($drive);
2517
2518     return undef if drive_is_cdrom($drive);
2519
2520     my $volid = $drive->{file};
2521     return undef if !$volid;
2522
2523     return $drive->{size};
2524 }
2525
2526 my $last_proc_pid_stat;
2527
2528 # get VM status information
2529 # This must be fast and should not block ($full == false)
2530 # We only query KVM using QMP if $full == true (this can be slow)
2531 sub vmstatus {
2532     my ($opt_vmid, $full) = @_;
2533
2534     my $res = {};
2535
2536     my $storecfg = PVE::Storage::config();
2537
2538     my $list = vzlist();
2539     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2540
2541     my $cpucount = $cpuinfo->{cpus} || 1;
2542
2543     foreach my $vmid (keys %$list) {
2544         next if $opt_vmid && ($vmid ne $opt_vmid);
2545
2546         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2547         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2548
2549         my $d = {};
2550         $d->{pid} = $list->{$vmid}->{pid};
2551
2552         # fixme: better status?
2553         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2554
2555         my $size = disksize($storecfg, $conf);
2556         if (defined($size)) {
2557             $d->{disk} = 0; # no info available
2558             $d->{maxdisk} = $size;
2559         } else {
2560             $d->{disk} = 0;
2561             $d->{maxdisk} = 0;
2562         }
2563
2564         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2565         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2566         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2567
2568         $d->{name} = $conf->{name} || "VM $vmid";
2569         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2570
2571         if ($conf->{balloon}) {
2572             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2573             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2574         }
2575
2576         $d->{uptime} = 0;
2577         $d->{cpu} = 0;
2578         $d->{mem} = 0;
2579
2580         $d->{netout} = 0;
2581         $d->{netin} = 0;
2582
2583         $d->{diskread} = 0;
2584         $d->{diskwrite} = 0;
2585
2586         $d->{template} = PVE::QemuConfig->is_template($conf);
2587
2588         $res->{$vmid} = $d;
2589     }
2590
2591     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2592     foreach my $dev (keys %$netdev) {
2593         next if $dev !~ m/^tap([1-9]\d*)i/;
2594         my $vmid = $1;
2595         my $d = $res->{$vmid};
2596         next if !$d;
2597
2598         $d->{netout} += $netdev->{$dev}->{receive};
2599         $d->{netin} += $netdev->{$dev}->{transmit};
2600
2601         if ($full) {
2602             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2603             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2604         }
2605
2606     }
2607
2608     my $ctime = gettimeofday;
2609
2610     foreach my $vmid (keys %$list) {
2611
2612         my $d = $res->{$vmid};
2613         my $pid = $d->{pid};
2614         next if !$pid;
2615
2616         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2617         next if !$pstat; # not running
2618
2619         my $used = $pstat->{utime} + $pstat->{stime};
2620
2621         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2622
2623         if ($pstat->{vsize}) {
2624             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2625         }
2626
2627         my $old = $last_proc_pid_stat->{$pid};
2628         if (!$old) {
2629             $last_proc_pid_stat->{$pid} = {
2630                 time => $ctime,
2631                 used => $used,
2632                 cpu => 0,
2633             };
2634             next;
2635         }
2636
2637         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2638
2639         if ($dtime > 1000) {
2640             my $dutime = $used -  $old->{used};
2641
2642             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2643             $last_proc_pid_stat->{$pid} = {
2644                 time => $ctime,
2645                 used => $used,
2646                 cpu => $d->{cpu},
2647             };
2648         } else {
2649             $d->{cpu} = $old->{cpu};
2650         }
2651     }
2652
2653     return $res if !$full;
2654
2655     my $qmpclient = PVE::QMPClient->new();
2656
2657     my $ballooncb = sub {
2658         my ($vmid, $resp) = @_;
2659
2660         my $info = $resp->{'return'};
2661         return if !$info->{max_mem};
2662
2663         my $d = $res->{$vmid};
2664
2665         # use memory assigned to VM
2666         $d->{maxmem} = $info->{max_mem};
2667         $d->{balloon} = $info->{actual};
2668
2669         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2670             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2671             $d->{freemem} = $info->{free_mem};
2672         }
2673
2674         $d->{ballooninfo} = $info;
2675     };
2676
2677     my $blockstatscb = sub {
2678         my ($vmid, $resp) = @_;
2679         my $data = $resp->{'return'} || [];
2680         my $totalrdbytes = 0;
2681         my $totalwrbytes = 0;
2682
2683         for my $blockstat (@$data) {
2684             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2685             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2686
2687             $blockstat->{device} =~ s/drive-//;
2688             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2689         }
2690         $res->{$vmid}->{diskread} = $totalrdbytes;
2691         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2692     };
2693
2694     my $statuscb = sub {
2695         my ($vmid, $resp) = @_;
2696
2697         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2698         # this fails if ballon driver is not loaded, so this must be
2699         # the last commnand (following command are aborted if this fails).
2700         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2701
2702         my $status = 'unknown';
2703         if (!defined($status = $resp->{'return'}->{status})) {
2704             warn "unable to get VM status\n";
2705             return;
2706         }
2707
2708         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2709     };
2710
2711     foreach my $vmid (keys %$list) {
2712         next if $opt_vmid && ($vmid ne $opt_vmid);
2713         next if !$res->{$vmid}->{pid}; # not running
2714         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2715     }
2716
2717     $qmpclient->queue_execute(undef, 2);
2718
2719     foreach my $vmid (keys %$list) {
2720         next if $opt_vmid && ($vmid ne $opt_vmid);
2721         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2722     }
2723
2724     return $res;
2725 }
2726
2727 sub foreach_drive {
2728     my ($conf, $func, @param) = @_;
2729
2730     foreach my $ds (valid_drive_names()) {
2731         next if !defined($conf->{$ds});
2732
2733         my $drive = parse_drive($ds, $conf->{$ds});
2734         next if !$drive;
2735
2736         &$func($ds, $drive, @param);
2737     }
2738 }
2739
2740 sub foreach_volid {
2741     my ($conf, $func, @param) = @_;
2742
2743     my $volhash = {};
2744
2745     my $test_volid = sub {
2746         my ($volid, $is_cdrom) = @_;
2747
2748         return if !$volid;
2749
2750         $volhash->{$volid} = $is_cdrom || 0;
2751     };
2752
2753     foreach_drive($conf, sub {
2754         my ($ds, $drive) = @_;
2755         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2756     });
2757
2758     foreach my $snapname (keys %{$conf->{snapshots}}) {
2759         my $snap = $conf->{snapshots}->{$snapname};
2760         &$test_volid($snap->{vmstate}, 0);
2761         foreach_drive($snap, sub {
2762             my ($ds, $drive) = @_;
2763             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2764         });
2765     }
2766
2767     foreach my $volid (keys %$volhash) {
2768         &$func($volid, $volhash->{$volid}, @param);
2769     }
2770 }
2771
2772 sub vga_conf_has_spice {
2773     my ($vga) = @_;
2774
2775     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2776
2777     return $1 || 1;
2778 }
2779
2780 sub config_to_command {
2781     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2782
2783     my $cmd = [];
2784     my $globalFlags = [];
2785     my $machineFlags = [];
2786     my $rtcFlags = [];
2787     my $cpuFlags = [];
2788     my $devices = [];
2789     my $pciaddr = '';
2790     my $bridges = {};
2791     my $kvmver = kvm_user_version();
2792     my $vernum = 0; # unknown
2793     my $ostype = $conf->{ostype};
2794     my $winversion = windows_version($ostype);
2795
2796     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2797         $vernum = $1*1000000+$2*1000;
2798     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2799         $vernum = $1*1000000+$2*1000+$3;
2800     }
2801
2802     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2803
2804     my $have_ovz = -f '/proc/vz/vestat';
2805
2806     my $q35 = machine_type_is_q35($conf);
2807     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2808     my $machine_type = $forcemachine || $conf->{machine};
2809     my $use_old_bios_files = undef;
2810     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2811
2812     my $cpuunits = defined($conf->{cpuunits}) ?
2813             $conf->{cpuunits} : $defaults->{cpuunits};
2814
2815     push @$cmd, '/usr/bin/kvm';
2816
2817     push @$cmd, '-id', $vmid;
2818
2819     my $use_virtio = 0;
2820
2821     my $qmpsocket = qmp_socket($vmid);
2822     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2823     push @$cmd, '-mon', "chardev=qmp,mode=control";
2824
2825
2826     push @$cmd, '-pidfile' , pidfile_name($vmid);
2827
2828     push @$cmd, '-daemonize';
2829
2830     if ($conf->{smbios1}) {
2831         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2832     }
2833
2834     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2835         my $ovmfbase;
2836
2837         # prefer the OVMF_CODE variant
2838         if (-f $OVMF_CODE) {
2839             $ovmfbase = $OVMF_CODE;
2840         } elsif (-f $OVMF_IMG) {
2841             $ovmfbase = $OVMF_IMG;
2842         }
2843
2844         die "no uefi base img found\n" if !$ovmfbase;
2845         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2846
2847         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2848             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2849             my $format = $d->{format} // 'raw';
2850             my $path;
2851             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2852             if ($storeid) {
2853                 $path = PVE::Storage::path($storecfg, $d->{file});
2854                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2855                 $format = qemu_img_format($scfg, $volname);
2856             } else {
2857                 $path = $d->{file};
2858                 $format = "raw";
2859             }
2860             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2861         } elsif ($ovmfbase eq $OVMF_CODE) {
2862             warn "using uefi without permanent efivars disk\n";
2863             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2864             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2865             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2866         } else {
2867             # if the base img is not OVMF_CODE, we do not have to bother
2868             # to create/use a vars image, since it will not be used anyway
2869             # this can only happen if someone manually deletes the OVMF_CODE image
2870             # or has an old pve-qemu-kvm version installed.
2871             # both should not happen, but we ignore it here
2872         }
2873     }
2874
2875
2876     # add usb controllers
2877     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2878     push @$devices, @usbcontrollers if @usbcontrollers;
2879     my $vga = $conf->{vga};
2880
2881     my $qxlnum = vga_conf_has_spice($vga);
2882     $vga = 'qxl' if $qxlnum;
2883
2884     if (!$vga) {
2885         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2886     }
2887
2888     # enable absolute mouse coordinates (needed by vnc)
2889     my $tablet;
2890     if (defined($conf->{tablet})) {
2891         $tablet = $conf->{tablet};
2892     } else {
2893         $tablet = $defaults->{tablet};
2894         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2895         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2896     }
2897
2898     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2899
2900     my $kvm_off = 0;
2901     my $gpu_passthrough;
2902
2903     # host pci devices
2904     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2905         my $d = parse_hostpci($conf->{"hostpci$i"});
2906         next if !$d;
2907
2908         my $pcie = $d->{pcie};
2909         if($pcie){
2910             die "q35 machine model is not enabled" if !$q35;
2911             $pciaddr = print_pcie_addr("hostpci$i");
2912         }else{
2913             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2914         }
2915
2916         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2917         my $romfile = $d->{romfile};
2918
2919         my $xvga = '';
2920         if ($d->{'x-vga'}) {
2921             $xvga = ',x-vga=on';
2922             $kvm_off = 1;
2923             $vga = 'none';
2924             $gpu_passthrough = 1;
2925
2926             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2927                 $xvga = "";
2928             }
2929         }
2930         my $pcidevices = $d->{pciid};
2931         my $multifunction = 1 if @$pcidevices > 1;
2932
2933         my $j=0;
2934         foreach my $pcidevice (@$pcidevices) {
2935
2936             my $id = "hostpci$i";
2937             $id .= ".$j" if $multifunction;
2938             my $addr = $pciaddr;
2939             $addr .= ".$j" if $multifunction;
2940             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2941
2942             if($j == 0){
2943                 $devicestr .= "$rombar$xvga";
2944                 $devicestr .= ",multifunction=on" if $multifunction;
2945                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2946             }
2947
2948             push @$devices, '-device', $devicestr;
2949             $j++;
2950         }
2951     }
2952
2953     # usb devices
2954     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2955     push @$devices, @usbdevices if @usbdevices;
2956     # serial devices
2957     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2958         if (my $path = $conf->{"serial$i"}) {
2959             if ($path eq 'socket') {
2960                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2961                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2962                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2963             } else {
2964                 die "no such serial device\n" if ! -c $path;
2965                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2966                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2967             }
2968         }
2969     }
2970
2971     # parallel devices
2972     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2973         if (my $path = $conf->{"parallel$i"}) {
2974             die "no such parallel device\n" if ! -c $path;
2975             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2976             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2977             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2978         }
2979     }
2980
2981     my $vmname = $conf->{name} || "vm$vmid";
2982
2983     push @$cmd, '-name', $vmname;
2984
2985     my $sockets = 1;
2986     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2987     $sockets = $conf->{sockets} if  $conf->{sockets};
2988
2989     my $cores = $conf->{cores} || 1;
2990
2991     my $maxcpus = $sockets * $cores;
2992
2993     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2994
2995     my $allowed_vcpus = $cpuinfo->{cpus};
2996
2997     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2998         if ($allowed_vcpus < $maxcpus);
2999
3000     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3001
3002         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3003         for (my $i = 2; $i <= $vcpus; $i++)  {
3004             my $cpustr = print_cpu_device($conf,$i);
3005             push @$cmd, '-device', $cpustr;
3006         }
3007
3008     } else {
3009
3010         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3011     }
3012     push @$cmd, '-nodefaults';
3013
3014     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3015
3016     my $bootindex_hash = {};
3017     my $i = 1;
3018     foreach my $o (split(//, $bootorder)) {
3019         $bootindex_hash->{$o} = $i*100;
3020         $i++;
3021     }
3022
3023     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3024
3025     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3026
3027     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3028
3029     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3030
3031     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3032         my $socket = vnc_socket($vmid);
3033         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3034     } else {
3035         push @$cmd, '-nographic';
3036     }
3037
3038     # time drift fix
3039     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3040
3041     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3042     my $useLocaltime = $conf->{localtime};
3043
3044     if ($winversion >= 5) { # windows
3045         $useLocaltime = 1 if !defined($conf->{localtime});
3046
3047         # use time drift fix when acpi is enabled
3048         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3049             $tdf = 1 if !defined($conf->{tdf});
3050         }
3051     }
3052
3053     if ($winversion >= 6) {
3054         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3055         push @$cmd, '-no-hpet';
3056     }
3057
3058     push @$rtcFlags, 'driftfix=slew' if $tdf;
3059
3060     if ($nokvm) {
3061         push @$machineFlags, 'accel=tcg';
3062     } else {
3063         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3064     }
3065
3066     if ($machine_type) {
3067         push @$machineFlags, "type=${machine_type}";
3068     }
3069
3070     if ($conf->{startdate}) {
3071         push @$rtcFlags, "base=$conf->{startdate}";
3072     } elsif ($useLocaltime) {
3073         push @$rtcFlags, 'base=localtime';
3074     }
3075
3076     my $cpu = $nokvm ? "qemu64" : "kvm64";
3077     if (my $cputype = $conf->{cpu}) {
3078         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3079             or die "Cannot parse cpu description: $cputype\n";
3080         $cpu = $cpuconf->{cputype};
3081         $kvm_off = 1 if $cpuconf->{hidden};
3082     }
3083
3084     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3085
3086     push @$cpuFlags , '-x2apic'
3087         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3088
3089     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3090
3091     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3092
3093     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3094
3095         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3096         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3097     }
3098
3099     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3100
3101     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3102
3103     push @$cpuFlags, 'kvm=off' if $kvm_off;
3104
3105     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3106         die "internal error"; # should not happen
3107
3108     push @$cpuFlags, "vendor=${cpu_vendor}"
3109         if $cpu_vendor ne 'default';
3110
3111     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3112
3113     push @$cmd, '-cpu', $cpu;
3114
3115     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3116
3117     push @$cmd, '-S' if $conf->{freeze};
3118
3119     # set keyboard layout
3120     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3121     push @$cmd, '-k', $kb if $kb;
3122
3123     # enable sound
3124     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3125     #push @$cmd, '-soundhw', 'es1370';
3126     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3127
3128     if($conf->{agent}) {
3129         my $qgasocket = qmp_socket($vmid, 1);
3130         my $pciaddr = print_pci_addr("qga0", $bridges);
3131         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3132         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3133         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3134     }
3135
3136     my $spice_port;
3137
3138     if ($qxlnum) {
3139         if ($qxlnum > 1) {
3140             if ($winversion){
3141                 for(my $i = 1; $i < $qxlnum; $i++){
3142                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3143                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3144                 }
3145             } else {
3146                 # assume other OS works like Linux
3147                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3148                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3149             }
3150         }
3151
3152         my $pciaddr = print_pci_addr("spice", $bridges);
3153
3154         my $nodename = PVE::INotify::nodename();
3155         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3156         $spice_port = PVE::Tools::next_spice_port($pfamily);
3157
3158         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3159
3160         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3161         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3162         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3163     }
3164
3165     # enable balloon by default, unless explicitly disabled
3166     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3167         $pciaddr = print_pci_addr("balloon0", $bridges);
3168         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3169     }
3170
3171     if ($conf->{watchdog}) {
3172         my $wdopts = parse_watchdog($conf->{watchdog});
3173         $pciaddr = print_pci_addr("watchdog", $bridges);
3174         my $watchdog = $wdopts->{model} || 'i6300esb';
3175         push @$devices, '-device', "$watchdog$pciaddr";
3176         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3177     }
3178
3179     my $vollist = [];
3180     my $scsicontroller = {};
3181     my $ahcicontroller = {};
3182     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3183
3184     # Add iscsi initiator name if available
3185     if (my $initiator = get_initiator_name()) {
3186         push @$devices, '-iscsi', "initiator-name=$initiator";
3187     }
3188
3189     foreach_drive($conf, sub {
3190         my ($ds, $drive) = @_;
3191
3192         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3193             push @$vollist, $drive->{file};
3194         }
3195
3196         $use_virtio = 1 if $ds =~ m/^virtio/;
3197
3198         if (drive_is_cdrom ($drive)) {
3199             if ($bootindex_hash->{d}) {
3200                 $drive->{bootindex} = $bootindex_hash->{d};
3201                 $bootindex_hash->{d} += 1;
3202             }
3203         } else {
3204             if ($bootindex_hash->{c}) {
3205                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3206                 $bootindex_hash->{c} += 1;
3207             }
3208         }
3209
3210         if($drive->{interface} eq 'virtio'){
3211            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3212         }
3213
3214         if ($drive->{interface} eq 'scsi') {
3215
3216             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3217
3218             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3219             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3220
3221             my $iothread = '';
3222             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3223                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3224                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3225             } elsif ($drive->{iothread}) {
3226                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3227             }
3228
3229             my $queues = '';
3230             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3231                 $queues = ",num_queues=$drive->{queues}";
3232             }
3233
3234             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3235             $scsicontroller->{$controller}=1;
3236         }
3237
3238         if ($drive->{interface} eq 'sata') {
3239            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3240            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3241            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3242            $ahcicontroller->{$controller}=1;
3243         }
3244
3245         if ($drive->{interface} eq 'efidisk') {
3246             # this will be added somewhere else
3247             return;
3248         }
3249
3250         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3251         push @$devices, '-drive',$drive_cmd;
3252         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3253     });
3254
3255     for (my $i = 0; $i < $MAX_NETS; $i++) {
3256          next if !$conf->{"net$i"};
3257          my $d = parse_net($conf->{"net$i"});
3258          next if !$d;
3259
3260          $use_virtio = 1 if $d->{model} eq 'virtio';
3261
3262          if ($bootindex_hash->{n}) {
3263             $d->{bootindex} = $bootindex_hash->{n};
3264             $bootindex_hash->{n} += 1;
3265          }
3266
3267          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3268          push @$devices, '-netdev', $netdevfull;
3269
3270          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3271          push @$devices, '-device', $netdevicefull;
3272     }
3273
3274     if (!$q35) {
3275         # add pci bridges
3276         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3277            $bridges->{1} = 1;
3278            $bridges->{2} = 1;
3279         }
3280
3281         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3282
3283         while (my ($k, $v) = each %$bridges) {
3284             $pciaddr = print_pci_addr("pci.$k");
3285             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3286         }
3287     }
3288
3289     # add custom args
3290     if ($conf->{args}) {
3291         my $aa = PVE::Tools::split_args($conf->{args});
3292         push @$cmd, @$aa;
3293     }
3294
3295     push @$cmd, @$devices;
3296     push @$cmd, '-rtc', join(',', @$rtcFlags)
3297         if scalar(@$rtcFlags);
3298     push @$cmd, '-machine', join(',', @$machineFlags)
3299         if scalar(@$machineFlags);
3300     push @$cmd, '-global', join(',', @$globalFlags)
3301         if scalar(@$globalFlags);
3302
3303     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3304 }
3305
3306 sub vnc_socket {
3307     my ($vmid) = @_;
3308     return "${var_run_tmpdir}/$vmid.vnc";
3309 }
3310
3311 sub spice_port {
3312     my ($vmid) = @_;
3313
3314     my $res = vm_mon_cmd($vmid, 'query-spice');
3315
3316     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3317 }
3318
3319 sub qmp_socket {
3320     my ($vmid, $qga) = @_;
3321     my $sockettype = $qga ? 'qga' : 'qmp';
3322     return "${var_run_tmpdir}/$vmid.$sockettype";
3323 }
3324
3325 sub pidfile_name {
3326     my ($vmid) = @_;
3327     return "${var_run_tmpdir}/$vmid.pid";
3328 }
3329
3330 sub vm_devices_list {
3331     my ($vmid) = @_;
3332
3333     my $res = vm_mon_cmd($vmid, 'query-pci');
3334     my $devices = {};
3335     foreach my $pcibus (@$res) {
3336         foreach my $device (@{$pcibus->{devices}}) {
3337             next if !$device->{'qdev_id'};
3338             if ($device->{'pci_bridge'}) {
3339                 $devices->{$device->{'qdev_id'}} = 1;
3340                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3341                     next if !$bridge_device->{'qdev_id'};
3342                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3343                     $devices->{$device->{'qdev_id'}}++;
3344                 }
3345             } else {
3346                 $devices->{$device->{'qdev_id'}} = 1;
3347             }
3348         }
3349     }
3350
3351     my $resblock = vm_mon_cmd($vmid, 'query-block');
3352     foreach my $block (@$resblock) {
3353         if($block->{device} =~ m/^drive-(\S+)/){
3354                 $devices->{$1} = 1;
3355         }
3356     }
3357
3358     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3359     foreach my $mice (@$resmice) {
3360         if ($mice->{name} eq 'QEMU HID Tablet') {
3361             $devices->{tablet} = 1;
3362             last;
3363         }
3364     }
3365
3366     # for usb devices there is no query-usb
3367     # but we can iterate over the entries in
3368     # qom-list path=/machine/peripheral
3369     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3370     foreach my $per (@$resperipheral) {
3371         if ($per->{name} =~ m/^usb\d+$/) {
3372             $devices->{$per->{name}} = 1;
3373         }
3374     }
3375
3376     return $devices;
3377 }
3378
3379 sub vm_deviceplug {
3380     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3381
3382     my $q35 = machine_type_is_q35($conf);
3383
3384     my $devices_list = vm_devices_list($vmid);
3385     return 1 if defined($devices_list->{$deviceid});
3386
3387     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3388
3389     if ($deviceid eq 'tablet') {
3390
3391         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3392
3393     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3394
3395         die "usb hotplug currently not reliable\n";
3396         # since we can't reliably hot unplug all added usb devices
3397         # and usb passthrough disables live migration
3398         # we disable usb hotplugging for now
3399         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3400
3401     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3402
3403         qemu_iothread_add($vmid, $deviceid, $device);
3404
3405         qemu_driveadd($storecfg, $vmid, $device);
3406         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3407
3408         qemu_deviceadd($vmid, $devicefull);
3409         eval { qemu_deviceaddverify($vmid, $deviceid); };
3410         if (my $err = $@) {
3411             eval { qemu_drivedel($vmid, $deviceid); };
3412             warn $@ if $@;
3413             die $err;
3414         }
3415
3416     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3417
3418
3419         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3420         my $pciaddr = print_pci_addr($deviceid);
3421         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3422
3423         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3424
3425         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3426             qemu_iothread_add($vmid, $deviceid, $device);
3427             $devicefull .= ",iothread=iothread-$deviceid";
3428         }
3429
3430         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3431             $devicefull .= ",num_queues=$device->{queues}";
3432         }
3433
3434         qemu_deviceadd($vmid, $devicefull);
3435         qemu_deviceaddverify($vmid, $deviceid);
3436
3437     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3438
3439         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3440         qemu_driveadd($storecfg, $vmid, $device);
3441
3442         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3443         eval { qemu_deviceadd($vmid, $devicefull); };
3444         if (my $err = $@) {
3445             eval { qemu_drivedel($vmid, $deviceid); };
3446             warn $@ if $@;
3447             die $err;
3448         }
3449
3450     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3451
3452         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3453
3454         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3455         my $use_old_bios_files = undef;
3456         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3457
3458         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3459         qemu_deviceadd($vmid, $netdevicefull);
3460         eval { qemu_deviceaddverify($vmid, $deviceid); };
3461         if (my $err = $@) {
3462             eval { qemu_netdevdel($vmid, $deviceid); };
3463             warn $@ if $@;
3464             die $err;
3465         }
3466
3467     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3468
3469         my $bridgeid = $2;
3470         my $pciaddr = print_pci_addr($deviceid);
3471         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3472
3473         qemu_deviceadd($vmid, $devicefull);
3474         qemu_deviceaddverify($vmid, $deviceid);
3475
3476     } else {
3477         die "can't hotplug device '$deviceid'\n";
3478     }
3479
3480     return 1;
3481 }
3482
3483 # fixme: this should raise exceptions on error!
3484 sub vm_deviceunplug {
3485     my ($vmid, $conf, $deviceid) = @_;
3486
3487     my $devices_list = vm_devices_list($vmid);
3488     return 1 if !defined($devices_list->{$deviceid});
3489
3490     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3491
3492     if ($deviceid eq 'tablet') {
3493
3494         qemu_devicedel($vmid, $deviceid);
3495
3496     } elsif ($deviceid =~ m/^usb\d+$/) {
3497
3498         die "usb hotplug currently not reliable\n";
3499         # when unplugging usb devices this way,
3500         # there may be remaining usb controllers/hubs
3501         # so we disable it for now
3502         qemu_devicedel($vmid, $deviceid);
3503         qemu_devicedelverify($vmid, $deviceid);
3504
3505     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3506
3507         qemu_devicedel($vmid, $deviceid);
3508         qemu_devicedelverify($vmid, $deviceid);
3509         qemu_drivedel($vmid, $deviceid);
3510         qemu_iothread_del($conf, $vmid, $deviceid);
3511
3512     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3513
3514         qemu_devicedel($vmid, $deviceid);
3515         qemu_devicedelverify($vmid, $deviceid);
3516         qemu_iothread_del($conf, $vmid, $deviceid);
3517
3518     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3519
3520         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3521         my $device = parse_drive($deviceid, $conf->{$deviceid});
3522         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3523
3524         qemu_devicedel($vmid, $deviceid);
3525         qemu_drivedel($vmid, $deviceid);
3526         qemu_deletescsihw($conf, $vmid, $deviceid);
3527
3528     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3529
3530         qemu_devicedel($vmid, $deviceid);
3531         qemu_devicedelverify($vmid, $deviceid);
3532         qemu_netdevdel($vmid, $deviceid);
3533
3534     } else {
3535         die "can't unplug device '$deviceid'\n";
3536     }
3537
3538     return 1;
3539 }
3540
3541 sub qemu_deviceadd {
3542     my ($vmid, $devicefull) = @_;
3543
3544     $devicefull = "driver=".$devicefull;
3545     my %options =  split(/[=,]/, $devicefull);
3546
3547     vm_mon_cmd($vmid, "device_add" , %options);
3548 }
3549
3550 sub qemu_devicedel {
3551     my ($vmid, $deviceid) = @_;
3552
3553     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3554 }
3555
3556 sub qemu_iothread_add {
3557     my($vmid, $deviceid, $device) = @_;
3558
3559     if ($device->{iothread}) {
3560         my $iothreads = vm_iothreads_list($vmid);
3561         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3562     }
3563 }
3564
3565 sub qemu_iothread_del {
3566     my($conf, $vmid, $deviceid) = @_;
3567
3568     my $device = parse_drive($deviceid, $conf->{$deviceid});
3569     if ($device->{iothread}) {
3570         my $iothreads = vm_iothreads_list($vmid);
3571         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3572     }
3573 }
3574
3575 sub qemu_objectadd {
3576     my($vmid, $objectid, $qomtype) = @_;
3577
3578     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3579
3580     return 1;
3581 }
3582
3583 sub qemu_objectdel {
3584     my($vmid, $objectid) = @_;
3585
3586     vm_mon_cmd($vmid, "object-del", id => $objectid);
3587
3588     return 1;
3589 }
3590
3591 sub qemu_driveadd {
3592     my ($storecfg, $vmid, $device) = @_;
3593
3594     my $drive = print_drive_full($storecfg, $vmid, $device);
3595     $drive =~ s/\\/\\\\/g;
3596     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3597
3598     # If the command succeeds qemu prints: "OK"
3599     return 1 if $ret =~ m/OK/s;
3600
3601     die "adding drive failed: $ret\n";
3602 }
3603
3604 sub qemu_drivedel {
3605     my($vmid, $deviceid) = @_;
3606
3607     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3608     $ret =~ s/^\s+//;
3609
3610     return 1 if $ret eq "";
3611
3612     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3613     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3614
3615     die "deleting drive $deviceid failed : $ret\n";
3616 }
3617
3618 sub qemu_deviceaddverify {
3619     my ($vmid, $deviceid) = @_;
3620
3621     for (my $i = 0; $i <= 5; $i++) {
3622          my $devices_list = vm_devices_list($vmid);
3623          return 1 if defined($devices_list->{$deviceid});
3624          sleep 1;
3625     }
3626
3627     die "error on hotplug device '$deviceid'\n";
3628 }
3629
3630
3631 sub qemu_devicedelverify {
3632     my ($vmid, $deviceid) = @_;
3633
3634     # need to verify that the device is correctly removed as device_del
3635     # is async and empty return is not reliable
3636
3637     for (my $i = 0; $i <= 5; $i++) {
3638          my $devices_list = vm_devices_list($vmid);
3639          return 1 if !defined($devices_list->{$deviceid});
3640          sleep 1;
3641     }
3642
3643     die "error on hot-unplugging device '$deviceid'\n";
3644 }
3645
3646 sub qemu_findorcreatescsihw {
3647     my ($storecfg, $conf, $vmid, $device) = @_;
3648
3649     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3650
3651     my $scsihwid="$controller_prefix$controller";
3652     my $devices_list = vm_devices_list($vmid);
3653
3654     if(!defined($devices_list->{$scsihwid})) {
3655         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3656     }
3657
3658     return 1;
3659 }
3660
3661 sub qemu_deletescsihw {
3662     my ($conf, $vmid, $opt) = @_;
3663
3664     my $device = parse_drive($opt, $conf->{$opt});
3665
3666     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3667         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3668         return 1;
3669     }
3670
3671     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3672
3673     my $devices_list = vm_devices_list($vmid);
3674     foreach my $opt (keys %{$devices_list}) {
3675         if (PVE::QemuServer::is_valid_drivename($opt)) {
3676             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3677             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3678                 return 1;
3679             }
3680         }
3681     }
3682
3683     my $scsihwid="scsihw$controller";
3684
3685     vm_deviceunplug($vmid, $conf, $scsihwid);
3686
3687     return 1;
3688 }
3689
3690 sub qemu_add_pci_bridge {
3691     my ($storecfg, $conf, $vmid, $device) = @_;
3692
3693     my $bridges = {};
3694
3695     my $bridgeid;
3696
3697     print_pci_addr($device, $bridges);
3698
3699     while (my ($k, $v) = each %$bridges) {
3700         $bridgeid = $k;
3701     }
3702     return 1 if !defined($bridgeid) || $bridgeid < 1;
3703
3704     my $bridge = "pci.$bridgeid";
3705     my $devices_list = vm_devices_list($vmid);
3706
3707     if (!defined($devices_list->{$bridge})) {
3708         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3709     }
3710
3711     return 1;
3712 }
3713
3714 sub qemu_set_link_status {
3715     my ($vmid, $device, $up) = @_;
3716
3717     vm_mon_cmd($vmid, "set_link", name => $device,
3718                up => $up ? JSON::true : JSON::false);
3719 }
3720
3721 sub qemu_netdevadd {
3722     my ($vmid, $conf, $device, $deviceid) = @_;
3723
3724     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3725     my %options =  split(/[=,]/, $netdev);
3726
3727     vm_mon_cmd($vmid, "netdev_add",  %options);
3728     return 1;
3729 }
3730
3731 sub qemu_netdevdel {
3732     my ($vmid, $deviceid) = @_;
3733
3734     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3735 }
3736
3737 sub qemu_usb_hotplug {
3738     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3739
3740     return if !$device;
3741
3742     # remove the old one first
3743     vm_deviceunplug($vmid, $conf, $deviceid);
3744
3745     # check if xhci controller is necessary and available
3746     if ($device->{usb3}) {
3747
3748         my $devicelist = vm_devices_list($vmid);
3749
3750         if (!$devicelist->{xhci}) {
3751             my $pciaddr = print_pci_addr("xhci");
3752             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3753         }
3754     }
3755     my $d = parse_usb_device($device->{host});
3756     $d->{usb3} = $device->{usb3};
3757
3758     # add the new one
3759     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3760 }
3761
3762 sub qemu_cpu_hotplug {
3763     my ($vmid, $conf, $vcpus) = @_;
3764
3765     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3766
3767     my $sockets = 1;
3768     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3769     $sockets = $conf->{sockets} if  $conf->{sockets};
3770     my $cores = $conf->{cores} || 1;
3771     my $maxcpus = $sockets * $cores;
3772
3773     $vcpus = $maxcpus if !$vcpus;
3774
3775     die "you can't add more vcpus than maxcpus\n"
3776         if $vcpus > $maxcpus;
3777
3778     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3779
3780     if ($vcpus < $currentvcpus) {
3781
3782         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3783
3784             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3785                 qemu_devicedel($vmid, "cpu$i");
3786                 my $retry = 0;
3787                 my $currentrunningvcpus = undef;
3788                 while (1) {
3789                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3790                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3791                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3792                     $retry++;
3793                     sleep 1;
3794                 }
3795                 #update conf after each succesfull cpu unplug
3796                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3797                 PVE::QemuConfig->write_config($vmid, $conf);
3798             }
3799         } else {
3800             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3801         }
3802
3803         return;
3804     }
3805
3806     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3807     die "vcpus in running vm does not match its configuration\n"
3808         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3809
3810     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3811
3812         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3813             my $cpustr = print_cpu_device($conf, $i);
3814             qemu_deviceadd($vmid, $cpustr);
3815
3816             my $retry = 0;
3817             my $currentrunningvcpus = undef;
3818             while (1) {
3819                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3820                 last if scalar(@{$currentrunningvcpus}) == $i;
3821                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3822                 sleep 1;
3823                 $retry++;
3824             }
3825             #update conf after each succesfull cpu hotplug
3826             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3827             PVE::QemuConfig->write_config($vmid, $conf);
3828         }
3829     } else {
3830
3831         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3832             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3833         }
3834     }
3835 }
3836
3837 sub qemu_block_set_io_throttle {
3838     my ($vmid, $deviceid,
3839         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3840         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3841         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3842         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3843
3844     return if !check_running($vmid) ;
3845
3846     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3847         bps => int($bps),
3848         bps_rd => int($bps_rd),
3849         bps_wr => int($bps_wr),
3850         iops => int($iops),
3851         iops_rd => int($iops_rd),
3852         iops_wr => int($iops_wr),
3853         bps_max => int($bps_max),
3854         bps_rd_max => int($bps_rd_max),
3855         bps_wr_max => int($bps_wr_max),
3856         iops_max => int($iops_max),
3857         iops_rd_max => int($iops_rd_max),
3858         iops_wr_max => int($iops_wr_max),
3859         bps_max_length => int($bps_max_length),
3860         bps_rd_max_length => int($bps_rd_max_length),
3861         bps_wr_max_length => int($bps_wr_max_length),
3862         iops_max_length => int($iops_max_length),
3863         iops_rd_max_length => int($iops_rd_max_length),
3864         iops_wr_max_length => int($iops_wr_max_length),
3865     );
3866
3867 }
3868
3869 # old code, only used to shutdown old VM after update
3870 sub __read_avail {
3871     my ($fh, $timeout) = @_;
3872
3873     my $sel = new IO::Select;
3874     $sel->add($fh);
3875
3876     my $res = '';
3877     my $buf;
3878
3879     my @ready;
3880     while (scalar (@ready = $sel->can_read($timeout))) {
3881         my $count;
3882         if ($count = $fh->sysread($buf, 8192)) {
3883             if ($buf =~ /^(.*)\(qemu\) $/s) {
3884                 $res .= $1;
3885                 last;
3886             } else {
3887                 $res .= $buf;
3888             }
3889         } else {
3890             if (!defined($count)) {
3891                 die "$!\n";
3892             }
3893             last;
3894         }
3895     }
3896
3897     die "monitor read timeout\n" if !scalar(@ready);
3898
3899     return $res;
3900 }
3901
3902 # old code, only used to shutdown old VM after update
3903 sub vm_monitor_command {
3904     my ($vmid, $cmdstr, $nocheck) = @_;
3905
3906     my $res;
3907
3908     eval {
3909         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3910
3911         my $sname = "${var_run_tmpdir}/$vmid.mon";
3912
3913         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3914             die "unable to connect to VM $vmid socket - $!\n";
3915
3916         my $timeout = 3;
3917
3918         # hack: migrate sometime blocks the monitor (when migrate_downtime
3919         # is set)
3920         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3921             $timeout = 60*60; # 1 hour
3922         }
3923
3924         # read banner;
3925         my $data = __read_avail($sock, $timeout);
3926
3927         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3928             die "got unexpected qemu monitor banner\n";
3929         }
3930
3931         my $sel = new IO::Select;
3932         $sel->add($sock);
3933
3934         if (!scalar(my @ready = $sel->can_write($timeout))) {
3935             die "monitor write error - timeout";
3936         }
3937
3938         my $fullcmd = "$cmdstr\r";
3939
3940         # syslog('info', "VM $vmid monitor command: $cmdstr");
3941
3942         my $b;
3943         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3944             die "monitor write error - $!";
3945         }
3946
3947         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3948
3949         $timeout = 20;
3950
3951         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3952             $timeout = 60*60; # 1 hour
3953         } elsif ($cmdstr =~ m/^(eject|change)/) {
3954             $timeout = 60; # note: cdrom mount command is slow
3955         }
3956         if ($res = __read_avail($sock, $timeout)) {
3957
3958             my @lines = split("\r?\n", $res);
3959
3960             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3961
3962             $res = join("\n", @lines);
3963             $res .= "\n";
3964         }
3965     };
3966
3967     my $err = $@;
3968
3969     if ($err) {
3970         syslog("err", "VM $vmid monitor command failed - $err");
3971         die $err;
3972     }
3973
3974     return $res;
3975 }
3976
3977 sub qemu_block_resize {
3978     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3979
3980     my $running = check_running($vmid);
3981
3982     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3983
3984     return if !$running;
3985
3986     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3987
3988 }
3989
3990 sub qemu_volume_snapshot {
3991     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3992
3993     my $running = check_running($vmid);
3994
3995     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3996         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3997     } else {
3998         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3999     }
4000 }
4001
4002 sub qemu_volume_snapshot_delete {
4003     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4004
4005     my $running = check_running($vmid);
4006
4007     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4008         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4009     } else {
4010         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4011     }
4012 }
4013
4014 sub set_migration_caps {
4015     my ($vmid) = @_;
4016
4017     my $cap_ref = [];
4018
4019     my $enabled_cap = {
4020         "auto-converge" => 1,
4021         "xbzrle" => 1,
4022         "x-rdma-pin-all" => 0,
4023         "zero-blocks" => 0,
4024         "compress" => 0
4025     };
4026
4027     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4028
4029     for my $supported_capability (@$supported_capabilities) {
4030         push @$cap_ref, {
4031             capability => $supported_capability->{capability},
4032             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4033         };
4034     }
4035
4036     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4037 }
4038
4039 my $fast_plug_option = {
4040     'lock' => 1,
4041     'name' => 1,
4042     'onboot' => 1,
4043     'shares' => 1,
4044     'startup' => 1,
4045     'description' => 1,
4046     'protection' => 1,
4047 };
4048
4049 # hotplug changes in [PENDING]
4050 # $selection hash can be used to only apply specified options, for
4051 # example: { cores => 1 } (only apply changed 'cores')
4052 # $errors ref is used to return error messages
4053 sub vmconfig_hotplug_pending {
4054     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4055
4056     my $defaults = load_defaults();
4057
4058     # commit values which do not have any impact on running VM first
4059     # Note: those option cannot raise errors, we we do not care about
4060     # $selection and always apply them.
4061
4062     my $add_error = sub {
4063         my ($opt, $msg) = @_;
4064         $errors->{$opt} = "hotplug problem - $msg";
4065     };
4066
4067     my $changes = 0;
4068     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4069         if ($fast_plug_option->{$opt}) {
4070             $conf->{$opt} = $conf->{pending}->{$opt};
4071             delete $conf->{pending}->{$opt};
4072             $changes = 1;
4073         }
4074     }
4075
4076     if ($changes) {
4077         PVE::QemuConfig->write_config($vmid, $conf);
4078         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4079     }
4080
4081     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4082
4083     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4084     while (my ($opt, $force) = each %$pending_delete_hash) {
4085         next if $selection && !$selection->{$opt};
4086         eval {
4087             if ($opt eq 'hotplug') {
4088                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4089             } elsif ($opt eq 'tablet') {
4090                 die "skip\n" if !$hotplug_features->{usb};
4091                 if ($defaults->{tablet}) {
4092                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4093                 } else {
4094                     vm_deviceunplug($vmid, $conf, $opt);
4095                 }
4096             } elsif ($opt =~ m/^usb\d+/) {
4097                 die "skip\n";
4098                 # since we cannot reliably hot unplug usb devices
4099                 # we are disabling it
4100                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4101                 vm_deviceunplug($vmid, $conf, $opt);
4102             } elsif ($opt eq 'vcpus') {
4103                 die "skip\n" if !$hotplug_features->{cpu};
4104                 qemu_cpu_hotplug($vmid, $conf, undef);
4105             } elsif ($opt eq 'balloon') {
4106                 # enable balloon device is not hotpluggable
4107                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4108             } elsif ($fast_plug_option->{$opt}) {
4109                 # do nothing
4110             } elsif ($opt =~ m/^net(\d+)$/) {
4111                 die "skip\n" if !$hotplug_features->{network};
4112                 vm_deviceunplug($vmid, $conf, $opt);
4113             } elsif (is_valid_drivename($opt)) {
4114                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4115                 vm_deviceunplug($vmid, $conf, $opt);
4116                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4117             } elsif ($opt =~ m/^memory$/) {
4118                 die "skip\n" if !$hotplug_features->{memory};
4119                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4120             } elsif ($opt eq 'cpuunits') {
4121                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4122             } elsif ($opt eq 'cpulimit') {
4123                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4124             } else {
4125                 die "skip\n";
4126             }
4127         };
4128         if (my $err = $@) {
4129             &$add_error($opt, $err) if $err ne "skip\n";
4130         } else {
4131             # save new config if hotplug was successful
4132             delete $conf->{$opt};
4133             vmconfig_undelete_pending_option($conf, $opt);
4134             PVE::QemuConfig->write_config($vmid, $conf);
4135             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4136         }
4137     }
4138
4139     foreach my $opt (keys %{$conf->{pending}}) {
4140         next if $selection && !$selection->{$opt};
4141         my $value = $conf->{pending}->{$opt};
4142         eval {
4143             if ($opt eq 'hotplug') {
4144                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4145             } elsif ($opt eq 'tablet') {
4146                 die "skip\n" if !$hotplug_features->{usb};
4147                 if ($value == 1) {
4148                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4149                 } elsif ($value == 0) {
4150                     vm_deviceunplug($vmid, $conf, $opt);
4151                 }
4152             } elsif ($opt =~ m/^usb\d+$/) {
4153                 die "skip\n";
4154                 # since we cannot reliably hot unplug usb devices
4155                 # we are disabling it
4156                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4157                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4158                 die "skip\n" if !$d;
4159                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4160             } elsif ($opt eq 'vcpus') {
4161                 die "skip\n" if !$hotplug_features->{cpu};
4162                 qemu_cpu_hotplug($vmid, $conf, $value);
4163             } elsif ($opt eq 'balloon') {
4164                 # enable/disable balloning device is not hotpluggable
4165                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4166                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4167                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4168
4169                 # allow manual ballooning if shares is set to zero
4170                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4171                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4172                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4173                 }
4174             } elsif ($opt =~ m/^net(\d+)$/) {
4175                 # some changes can be done without hotplug
4176                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4177                                     $vmid, $opt, $value);
4178             } elsif (is_valid_drivename($opt)) {
4179                 # some changes can be done without hotplug
4180                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4181                                      $vmid, $opt, $value, 1);
4182             } elsif ($opt =~ m/^memory$/) { #dimms
4183                 die "skip\n" if !$hotplug_features->{memory};
4184                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4185             } elsif ($opt eq 'cpuunits') {
4186                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4187             } elsif ($opt eq 'cpulimit') {
4188                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4189                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4190             } else {
4191                 die "skip\n";  # skip non-hot-pluggable options
4192             }
4193         };
4194         if (my $err = $@) {
4195             &$add_error($opt, $err) if $err ne "skip\n";
4196         } else {
4197             # save new config if hotplug was successful
4198             $conf->{$opt} = $value;
4199             delete $conf->{pending}->{$opt};
4200             PVE::QemuConfig->write_config($vmid, $conf);
4201             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4202         }
4203     }
4204 }
4205
4206 sub try_deallocate_drive {
4207     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4208
4209     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4210         my $volid = $drive->{file};
4211         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4212             my $sid = PVE::Storage::parse_volume_id($volid);
4213             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4214
4215             # check if the disk is really unused
4216             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4217                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4218             PVE::Storage::vdisk_free($storecfg, $volid);
4219             return 1;
4220         } else {
4221             # If vm is not owner of this disk remove from config
4222             return 1;
4223         }
4224     }
4225
4226     return undef;
4227 }
4228
4229 sub vmconfig_delete_or_detach_drive {
4230     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4231
4232     my $drive = parse_drive($opt, $conf->{$opt});
4233
4234     my $rpcenv = PVE::RPCEnvironment::get();
4235     my $authuser = $rpcenv->get_user();
4236
4237     if ($force) {
4238         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4239         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4240     } else {
4241         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4242     }
4243 }
4244
4245 sub vmconfig_apply_pending {
4246     my ($vmid, $conf, $storecfg) = @_;
4247
4248     # cold plug
4249
4250     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4251     while (my ($opt, $force) = each %$pending_delete_hash) {
4252         die "internal error" if $opt =~ m/^unused/;
4253         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4254         if (!defined($conf->{$opt})) {
4255             vmconfig_undelete_pending_option($conf, $opt);
4256             PVE::QemuConfig->write_config($vmid, $conf);
4257         } elsif (is_valid_drivename($opt)) {
4258             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4259             vmconfig_undelete_pending_option($conf, $opt);
4260             delete $conf->{$opt};
4261             PVE::QemuConfig->write_config($vmid, $conf);
4262         } else {
4263             vmconfig_undelete_pending_option($conf, $opt);
4264             delete $conf->{$opt};
4265             PVE::QemuConfig->write_config($vmid, $conf);
4266         }
4267     }
4268
4269     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4270
4271     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4272         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4273
4274         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4275             # skip if nothing changed
4276         } elsif (is_valid_drivename($opt)) {
4277             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4278                 if defined($conf->{$opt});
4279             $conf->{$opt} = $conf->{pending}->{$opt};
4280         } else {
4281             $conf->{$opt} = $conf->{pending}->{$opt};
4282         }
4283
4284         delete $conf->{pending}->{$opt};
4285         PVE::QemuConfig->write_config($vmid, $conf);
4286     }
4287 }
4288
4289 my $safe_num_ne = sub {
4290     my ($a, $b) = @_;
4291
4292     return 0 if !defined($a) && !defined($b);
4293     return 1 if !defined($a);
4294     return 1 if !defined($b);
4295
4296     return $a != $b;
4297 };
4298
4299 my $safe_string_ne = sub {
4300     my ($a, $b) = @_;
4301
4302     return 0 if !defined($a) && !defined($b);
4303     return 1 if !defined($a);
4304     return 1 if !defined($b);
4305
4306     return $a ne $b;
4307 };
4308
4309 sub vmconfig_update_net {
4310     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4311
4312     my $newnet = parse_net($value);
4313
4314     if ($conf->{$opt}) {
4315         my $oldnet = parse_net($conf->{$opt});
4316
4317         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4318             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4319             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4320             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4321
4322             # for non online change, we try to hot-unplug
4323             die "skip\n" if !$hotplug;
4324             vm_deviceunplug($vmid, $conf, $opt);
4325         } else {
4326
4327             die "internal error" if $opt !~ m/net(\d+)/;
4328             my $iface = "tap${vmid}i$1";
4329
4330             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4331                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4332                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4333                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4334                 PVE::Network::tap_unplug($iface);
4335                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4336             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4337                 # Rate can be applied on its own but any change above needs to
4338                 # include the rate in tap_plug since OVS resets everything.
4339                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4340             }
4341
4342             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4343                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4344             }
4345
4346             return 1;
4347         }
4348     }
4349
4350     if ($hotplug) {
4351         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4352     } else {
4353         die "skip\n";
4354     }
4355 }
4356
4357 sub vmconfig_update_disk {
4358     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4359
4360     # fixme: do we need force?
4361
4362     my $drive = parse_drive($opt, $value);
4363
4364     if ($conf->{$opt}) {
4365
4366         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4367
4368             my $media = $drive->{media} || 'disk';
4369             my $oldmedia = $old_drive->{media} || 'disk';
4370             die "unable to change media type\n" if $media ne $oldmedia;
4371
4372             if (!drive_is_cdrom($old_drive)) {
4373
4374                 if ($drive->{file} ne $old_drive->{file}) {
4375
4376                     die "skip\n" if !$hotplug;
4377
4378                     # unplug and register as unused
4379                     vm_deviceunplug($vmid, $conf, $opt);
4380                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4381
4382                 } else {
4383                     # update existing disk
4384
4385                     # skip non hotpluggable value
4386                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4387                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4388                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4389                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4390                         die "skip\n";
4391                     }
4392
4393                     # apply throttle
4394                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4395                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4396                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4397                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4398                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4399                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4400                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4401                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4402                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4403                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4404                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4405                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4406                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4407                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4408                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4409                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4410                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4411                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4412
4413                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4414                                                    ($drive->{mbps} || 0)*1024*1024,
4415                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4416                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4417                                                    $drive->{iops} || 0,
4418                                                    $drive->{iops_rd} || 0,
4419                                                    $drive->{iops_wr} || 0,
4420                                                    ($drive->{mbps_max} || 0)*1024*1024,
4421                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4422                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4423                                                    $drive->{iops_max} || 0,
4424                                                    $drive->{iops_rd_max} || 0,
4425                                                    $drive->{iops_wr_max} || 0,
4426                                                    $drive->{bps_max_length} || 1,
4427                                                    $drive->{bps_rd_max_length} || 1,
4428                                                    $drive->{bps_wr_max_length} || 1,
4429                                                    $drive->{iops_max_length} || 1,
4430                                                    $drive->{iops_rd_max_length} || 1,
4431                                                    $drive->{iops_wr_max_length} || 1);
4432
4433                     }
4434
4435                     return 1;
4436                 }
4437
4438             } else { # cdrom
4439
4440                 if ($drive->{file} eq 'none') {
4441                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4442                 } else {
4443                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4444                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4445                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4446                 }
4447
4448                 return 1;
4449             }
4450         }
4451     }
4452
4453     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4454     # hotplug new disks
4455     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4456     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4457 }
4458
4459 sub vm_start {
4460     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4461         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4462
4463     PVE::QemuConfig->lock_config($vmid, sub {
4464         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4465
4466         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4467
4468         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4469
4470         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4471
4472         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4473             vmconfig_apply_pending($vmid, $conf, $storecfg);
4474             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4475         }
4476
4477         my $defaults = load_defaults();
4478
4479         # set environment variable useful inside network script
4480         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4481
4482         my $local_volumes = {};
4483
4484         if ($targetstorage) {
4485             foreach_drive($conf, sub {
4486                 my ($ds, $drive) = @_;
4487
4488                 return if drive_is_cdrom($drive);
4489
4490                 my $volid = $drive->{file};
4491
4492                 return if !$volid;
4493
4494                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4495
4496                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4497                 return if $scfg->{shared};
4498                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4499             });
4500
4501             my $format = undef;
4502
4503             foreach my $opt (sort keys %$local_volumes) {
4504
4505                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4506                 my $drive = parse_drive($opt, $conf->{$opt});
4507
4508                 #if remote storage is specified, use default format
4509                 if ($targetstorage && $targetstorage ne "1") {
4510                     $storeid = $targetstorage;
4511                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4512                     $format = $defFormat;
4513                 } else {
4514                     #else we use same format than original
4515                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4516                     $format = qemu_img_format($scfg, $volid);
4517                 }
4518
4519                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4520                 my $newdrive = $drive;
4521                 $newdrive->{format} = $format;
4522                 $newdrive->{file} = $newvolid;
4523                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4524                 $local_volumes->{$opt} = $drivestr;
4525                 #pass drive to conf for command line
4526                 $conf->{$opt} = $drivestr;
4527             }
4528         }
4529
4530         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4531
4532         my $migrate_port = 0;
4533         my $migrate_uri;
4534         if ($statefile) {
4535             if ($statefile eq 'tcp') {
4536                 my $localip = "localhost";
4537                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4538                 my $nodename = PVE::INotify::nodename();
4539
4540                 if (!defined($migration_type)) {
4541                     if (defined($datacenterconf->{migration}->{type})) {
4542                         $migration_type = $datacenterconf->{migration}->{type};
4543                     } else {
4544                         $migration_type = 'secure';
4545                     }
4546                 }
4547
4548                 if ($migration_type eq 'insecure') {
4549                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4550                     if ($migrate_network_addr) {
4551                         $localip = $migrate_network_addr;
4552                     } else {
4553                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4554                     }
4555
4556                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4557                 }
4558
4559                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4560                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4561                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4562                 push @$cmd, '-incoming', $migrate_uri;
4563                 push @$cmd, '-S';
4564
4565             } elsif ($statefile eq 'unix') {
4566                 # should be default for secure migrations as a ssh TCP forward
4567                 # tunnel is not deterministic reliable ready and fails regurarly
4568                 # to set up in time, so use UNIX socket forwards
4569                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4570                 unlink $socket_addr;
4571
4572                 $migrate_uri = "unix:$socket_addr";
4573
4574                 push @$cmd, '-incoming', $migrate_uri;
4575                 push @$cmd, '-S';
4576
4577             } else {
4578                 push @$cmd, '-loadstate', $statefile;
4579             }
4580         } elsif ($paused) {
4581             push @$cmd, '-S';
4582         }
4583
4584         # host pci devices
4585         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4586           my $d = parse_hostpci($conf->{"hostpci$i"});
4587           next if !$d;
4588           my $pcidevices = $d->{pciid};
4589           foreach my $pcidevice (@$pcidevices) {
4590                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4591
4592                 my $info = pci_device_info("0000:$pciid");
4593                 die "IOMMU not present\n" if !check_iommu_support();
4594                 die "no pci device info for device '$pciid'\n" if !$info;
4595                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4596                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4597           }
4598         }
4599
4600         PVE::Storage::activate_volumes($storecfg, $vollist);
4601
4602         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4603             my $cmd = [];
4604             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4605             eval  { run_command($cmd); };
4606         }
4607
4608         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4609                                                   : $defaults->{cpuunits};
4610
4611         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4612
4613         my %properties = (
4614             Slice => 'qemu.slice',
4615             KillMode => 'none',
4616             CPUShares => $cpuunits
4617         );
4618
4619         if (my $cpulimit = $conf->{cpulimit}) {
4620             $properties{CPUQuota} = int($cpulimit * 100);
4621         }
4622         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4623
4624         if ($conf->{hugepages}) {
4625
4626             my $code = sub {
4627                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4628                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4629
4630                 PVE::QemuServer::Memory::hugepages_mount();
4631                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4632
4633                 eval  {
4634                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4635                     run_command($cmd, %run_params);
4636                 };
4637
4638                 if (my $err = $@) {
4639                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4640                     die $err;
4641                 }
4642
4643                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4644             };
4645             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4646
4647         } else {
4648             eval  {
4649                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4650                 run_command($cmd, %run_params);
4651             };
4652         }
4653
4654         if (my $err = $@) {
4655             # deactivate volumes if start fails
4656             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4657             die "start failed: $err";
4658         }
4659
4660         print "migration listens on $migrate_uri\n" if $migrate_uri;
4661
4662         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4663             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4664             warn $@ if $@;
4665         }
4666
4667         #start nbd server for storage migration
4668         if ($targetstorage) {
4669             my $nodename = PVE::INotify::nodename();
4670             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4671             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4672             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4673             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4674
4675             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4676
4677             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4678
4679             foreach my $opt (sort keys %$local_volumes) {
4680                 my $volid = $local_volumes->{$opt};
4681                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4682                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4683                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4684             }
4685         }
4686
4687         if ($migratedfrom) {
4688             eval {
4689                 set_migration_caps($vmid);
4690             };
4691             warn $@ if $@;
4692
4693             if ($spice_port) {
4694                 print "spice listens on port $spice_port\n";
4695                 if ($spice_ticket) {
4696                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4697                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4698                 }
4699             }
4700
4701         } else {
4702             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4703                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4704                     if $conf->{balloon};
4705             }
4706
4707             foreach my $opt (keys %$conf) {
4708                 next if $opt !~  m/^net\d+$/;
4709                 my $nicconf = parse_net($conf->{$opt});
4710                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4711             }
4712         }
4713
4714         vm_mon_cmd_nocheck($vmid, 'qom-set',
4715                     path => "machine/peripheral/balloon0",
4716                     property => "guest-stats-polling-interval",
4717                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4718
4719     });
4720 }
4721
4722 sub vm_mon_cmd {
4723     my ($vmid, $execute, %params) = @_;
4724
4725     my $cmd = { execute => $execute, arguments => \%params };
4726     vm_qmp_command($vmid, $cmd);
4727 }
4728
4729 sub vm_mon_cmd_nocheck {
4730     my ($vmid, $execute, %params) = @_;
4731
4732     my $cmd = { execute => $execute, arguments => \%params };
4733     vm_qmp_command($vmid, $cmd, 1);
4734 }
4735
4736 sub vm_qmp_command {
4737     my ($vmid, $cmd, $nocheck) = @_;
4738
4739     my $res;
4740
4741     my $timeout;
4742     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4743         $timeout = $cmd->{arguments}->{timeout};
4744         delete $cmd->{arguments}->{timeout};
4745     }
4746
4747     eval {
4748         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4749         my $sname = qmp_socket($vmid);
4750         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4751             my $qmpclient = PVE::QMPClient->new();
4752
4753             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4754         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4755             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4756                 if scalar(%{$cmd->{arguments}});
4757             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4758         } else {
4759             die "unable to open monitor socket\n";
4760         }
4761     };
4762     if (my $err = $@) {
4763         syslog("err", "VM $vmid qmp command failed - $err");
4764         die $err;
4765     }
4766
4767     return $res;
4768 }
4769
4770 sub vm_human_monitor_command {
4771     my ($vmid, $cmdline) = @_;
4772
4773     my $res;
4774
4775     my $cmd = {
4776         execute => 'human-monitor-command',
4777         arguments => { 'command-line' => $cmdline},
4778     };
4779
4780     return vm_qmp_command($vmid, $cmd);
4781 }
4782
4783 sub vm_commandline {
4784     my ($storecfg, $vmid) = @_;
4785
4786     my $conf = PVE::QemuConfig->load_config($vmid);
4787
4788     my $defaults = load_defaults();
4789
4790     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4791
4792     return PVE::Tools::cmd2string($cmd);
4793 }
4794
4795 sub vm_reset {
4796     my ($vmid, $skiplock) = @_;
4797
4798     PVE::QemuConfig->lock_config($vmid, sub {
4799
4800         my $conf = PVE::QemuConfig->load_config($vmid);
4801
4802         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4803
4804         vm_mon_cmd($vmid, "system_reset");
4805     });
4806 }
4807
4808 sub get_vm_volumes {
4809     my ($conf) = @_;
4810
4811     my $vollist = [];
4812     foreach_volid($conf, sub {
4813         my ($volid, $is_cdrom) = @_;
4814
4815         return if $volid =~ m|^/|;
4816
4817         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4818         return if !$sid;
4819
4820         push @$vollist, $volid;
4821     });
4822
4823     return $vollist;
4824 }
4825
4826 sub vm_stop_cleanup {
4827     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4828
4829     eval {
4830
4831         if (!$keepActive) {
4832             my $vollist = get_vm_volumes($conf);
4833             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4834         }
4835
4836         foreach my $ext (qw(mon qmp pid vnc qga)) {
4837             unlink "/var/run/qemu-server/${vmid}.$ext";
4838         }
4839
4840         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4841     };
4842     warn $@ if $@; # avoid errors - just warn
4843 }
4844
4845 # Note: use $nockeck to skip tests if VM configuration file exists.
4846 # We need that when migration VMs to other nodes (files already moved)
4847 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4848 sub vm_stop {
4849     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4850
4851     $force = 1 if !defined($force) && !$shutdown;
4852
4853     if ($migratedfrom){
4854         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4855         kill 15, $pid if $pid;
4856         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4857         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4858         return;
4859     }
4860
4861     PVE::QemuConfig->lock_config($vmid, sub {
4862
4863         my $pid = check_running($vmid, $nocheck);
4864         return if !$pid;
4865
4866         my $conf;
4867         if (!$nocheck) {
4868             $conf = PVE::QemuConfig->load_config($vmid);
4869             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4870             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4871                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4872                 $timeout = $opts->{down} if $opts->{down};
4873             }
4874         }
4875
4876         $timeout = 60 if !defined($timeout);
4877
4878         eval {
4879             if ($shutdown) {
4880                 if (defined($conf) && $conf->{agent}) {
4881                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4882                 } else {
4883                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4884                 }
4885             } else {
4886                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4887             }
4888         };
4889         my $err = $@;
4890
4891         if (!$err) {
4892             my $count = 0;
4893             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4894                 $count++;
4895                 sleep 1;
4896             }
4897
4898             if ($count >= $timeout) {
4899                 if ($force) {
4900                     warn "VM still running - terminating now with SIGTERM\n";
4901                     kill 15, $pid;
4902                 } else {
4903                     die "VM quit/powerdown failed - got timeout\n";
4904                 }
4905             } else {
4906                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4907                 return;
4908             }
4909         } else {
4910             if ($force) {
4911                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4912                 kill 15, $pid;
4913             } else {
4914                 die "VM quit/powerdown failed\n";
4915             }
4916         }
4917
4918         # wait again
4919         $timeout = 10;
4920
4921         my $count = 0;
4922         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4923             $count++;
4924             sleep 1;
4925         }
4926
4927         if ($count >= $timeout) {
4928             warn "VM still running - terminating now with SIGKILL\n";
4929             kill 9, $pid;
4930             sleep 1;
4931         }
4932
4933         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4934    });
4935 }
4936
4937 sub vm_suspend {
4938     my ($vmid, $skiplock) = @_;
4939
4940     PVE::QemuConfig->lock_config($vmid, sub {
4941
4942         my $conf = PVE::QemuConfig->load_config($vmid);
4943
4944         PVE::QemuConfig->check_lock($conf)
4945             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4946
4947         vm_mon_cmd($vmid, "stop");
4948     });
4949 }
4950
4951 sub vm_resume {
4952     my ($vmid, $skiplock, $nocheck) = @_;
4953
4954     PVE::QemuConfig->lock_config($vmid, sub {
4955
4956         if (!$nocheck) {
4957
4958             my $conf = PVE::QemuConfig->load_config($vmid);
4959
4960             PVE::QemuConfig->check_lock($conf)
4961                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4962
4963             vm_mon_cmd($vmid, "cont");
4964
4965         } else {
4966             vm_mon_cmd_nocheck($vmid, "cont");
4967         }
4968     });
4969 }
4970
4971 sub vm_sendkey {
4972     my ($vmid, $skiplock, $key) = @_;
4973
4974     PVE::QemuConfig->lock_config($vmid, sub {
4975
4976         my $conf = PVE::QemuConfig->load_config($vmid);
4977
4978         # there is no qmp command, so we use the human monitor command
4979         vm_human_monitor_command($vmid, "sendkey $key");
4980     });
4981 }
4982
4983 sub vm_destroy {
4984     my ($storecfg, $vmid, $skiplock) = @_;
4985
4986     PVE::QemuConfig->lock_config($vmid, sub {
4987
4988         my $conf = PVE::QemuConfig->load_config($vmid);
4989
4990         if (!check_running($vmid)) {
4991             destroy_vm($storecfg, $vmid, undef, $skiplock);
4992         } else {
4993             die "VM $vmid is running - destroy failed\n";
4994         }
4995     });
4996 }
4997
4998 # pci helpers
4999
5000 sub file_write {
5001     my ($filename, $buf) = @_;
5002
5003     my $fh = IO::File->new($filename, "w");
5004     return undef if !$fh;
5005
5006     my $res = print $fh $buf;
5007
5008     $fh->close();
5009
5010     return $res;
5011 }
5012
5013 sub pci_device_info {
5014     my ($name) = @_;
5015
5016     my $res;
5017
5018     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5019     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5020
5021     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5022     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5023
5024     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5025     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5026
5027     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5028     return undef if !defined($product) || $product !~ s/^0x//;
5029
5030     $res = {
5031         name => $name,
5032         vendor => $vendor,
5033         product => $product,
5034         domain => $domain,
5035         bus => $bus,
5036         slot => $slot,
5037         func => $func,
5038         irq => $irq,
5039         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5040     };
5041
5042     return $res;
5043 }
5044
5045 sub pci_dev_reset {
5046     my ($dev) = @_;
5047
5048     my $name = $dev->{name};
5049
5050     my $fn = "$pcisysfs/devices/$name/reset";
5051
5052     return file_write($fn, "1");
5053 }
5054
5055 sub pci_dev_bind_to_vfio {
5056     my ($dev) = @_;
5057
5058     my $name = $dev->{name};
5059
5060     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5061
5062     if (!-d $vfio_basedir) {
5063         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5064     }
5065     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5066
5067     my $testdir = "$vfio_basedir/$name";
5068     return 1 if -d $testdir;
5069
5070     my $data = "$dev->{vendor} $dev->{product}";
5071     return undef if !file_write("$vfio_basedir/new_id", $data);
5072
5073     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5074     if (!file_write($fn, $name)) {
5075         return undef if -f $fn;
5076     }
5077
5078     $fn = "$vfio_basedir/bind";
5079     if (! -d $testdir) {
5080         return undef if !file_write($fn, $name);
5081     }
5082
5083     return -d $testdir;
5084 }
5085
5086 sub pci_dev_group_bind_to_vfio {
5087     my ($pciid) = @_;
5088
5089     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5090
5091     if (!-d $vfio_basedir) {
5092         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5093     }
5094     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5095
5096     # get IOMMU group devices
5097     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5098       my @devs = grep /^0000:/, readdir($D);
5099     closedir($D);
5100
5101     foreach my $pciid (@devs) {
5102         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5103
5104         # pci bridges, switches or root ports are not supported
5105         # they have a pci_bus subdirectory so skip them
5106         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5107
5108         my $info = pci_device_info($1);
5109         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5110     }
5111
5112     return 1;
5113 }
5114
5115 # vzdump restore implementaion
5116
5117 sub tar_archive_read_firstfile {
5118     my $archive = shift;
5119
5120     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5121
5122     # try to detect archive type first
5123     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5124         die "unable to open file '$archive'\n";
5125     my $firstfile = <$fh>;
5126     kill 15, $pid;
5127     close $fh;
5128
5129     die "ERROR: archive contaions no data\n" if !$firstfile;
5130     chomp $firstfile;
5131
5132     return $firstfile;
5133 }
5134
5135 sub tar_restore_cleanup {
5136     my ($storecfg, $statfile) = @_;
5137
5138     print STDERR "starting cleanup\n";
5139
5140     if (my $fd = IO::File->new($statfile, "r")) {
5141         while (defined(my $line = <$fd>)) {
5142             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5143                 my $volid = $2;
5144                 eval {
5145                     if ($volid =~ m|^/|) {
5146                         unlink $volid || die 'unlink failed\n';
5147                     } else {
5148                         PVE::Storage::vdisk_free($storecfg, $volid);
5149                     }
5150                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5151                 };
5152                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5153             } else {
5154                 print STDERR "unable to parse line in statfile - $line";
5155             }
5156         }
5157         $fd->close();
5158     }
5159 }
5160
5161 sub restore_archive {
5162     my ($archive, $vmid, $user, $opts) = @_;
5163
5164     my $format = $opts->{format};
5165     my $comp;
5166
5167     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5168         $format = 'tar' if !$format;
5169         $comp = 'gzip';
5170     } elsif ($archive =~ m/\.tar$/) {
5171         $format = 'tar' if !$format;
5172     } elsif ($archive =~ m/.tar.lzo$/) {
5173         $format = 'tar' if !$format;
5174         $comp = 'lzop';
5175     } elsif ($archive =~ m/\.vma$/) {
5176         $format = 'vma' if !$format;
5177     } elsif ($archive =~ m/\.vma\.gz$/) {
5178         $format = 'vma' if !$format;
5179         $comp = 'gzip';
5180     } elsif ($archive =~ m/\.vma\.lzo$/) {
5181         $format = 'vma' if !$format;
5182         $comp = 'lzop';
5183     } else {
5184         $format = 'vma' if !$format; # default
5185     }
5186
5187     # try to detect archive format
5188     if ($format eq 'tar') {
5189         return restore_tar_archive($archive, $vmid, $user, $opts);
5190     } else {
5191         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5192     }
5193 }
5194
5195 sub restore_update_config_line {
5196     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5197
5198     return if $line =~ m/^\#qmdump\#/;
5199     return if $line =~ m/^\#vzdump\#/;
5200     return if $line =~ m/^lock:/;
5201     return if $line =~ m/^unused\d+:/;
5202     return if $line =~ m/^parent:/;
5203     return if $line =~ m/^template:/; # restored VM is never a template
5204
5205     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5206     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5207         # try to convert old 1.X settings
5208         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5209         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5210             my ($model, $macaddr) = split(/\=/, $devconfig);
5211             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5212             my $net = {
5213                 model => $model,
5214                 bridge => "vmbr$ind",
5215                 macaddr => $macaddr,
5216             };
5217             my $netstr = print_net($net);
5218
5219             print $outfd "net$cookie->{netcount}: $netstr\n";
5220             $cookie->{netcount}++;
5221         }
5222     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5223         my ($id, $netstr) = ($1, $2);
5224         my $net = parse_net($netstr);
5225         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5226         $netstr = print_net($net);
5227         print $outfd "$id: $netstr\n";
5228     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5229         my $virtdev = $1;
5230         my $value = $3;
5231         my $di = parse_drive($virtdev, $value);
5232         if (defined($di->{backup}) && !$di->{backup}) {
5233             print $outfd "#$line";
5234         } elsif ($map->{$virtdev}) {
5235             delete $di->{format}; # format can change on restore
5236             $di->{file} = $map->{$virtdev};
5237             $value = print_drive($vmid, $di);
5238             print $outfd "$virtdev: $value\n";
5239         } else {
5240             print $outfd $line;
5241         }
5242     } else {
5243         print $outfd $line;
5244     }
5245 }
5246
5247 sub scan_volids {
5248     my ($cfg, $vmid) = @_;
5249
5250     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5251
5252     my $volid_hash = {};
5253     foreach my $storeid (keys %$info) {
5254         foreach my $item (@{$info->{$storeid}}) {
5255             next if !($item->{volid} && $item->{size});
5256             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5257             $volid_hash->{$item->{volid}} = $item;
5258         }
5259     }
5260
5261     return $volid_hash;
5262 }
5263
5264 sub is_volume_in_use {
5265     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5266
5267     my $path = PVE::Storage::path($storecfg, $volid);
5268
5269     my $scan_config = sub {
5270         my ($cref, $snapname) = @_;
5271
5272         foreach my $key (keys %$cref) {
5273             my $value = $cref->{$key};
5274             if (is_valid_drivename($key)) {
5275                 next if $skip_drive && $key eq $skip_drive;
5276                 my $drive = parse_drive($key, $value);
5277                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5278                 return 1 if $volid eq $drive->{file};
5279                 if ($drive->{file} =~ m!^/!) {
5280                     return 1 if $drive->{file} eq $path;
5281                 } else {
5282                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5283                     next if !$storeid;
5284                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5285                     next if !$scfg;
5286                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5287                 }
5288             }
5289         }
5290
5291         return 0;
5292     };
5293
5294     return 1 if &$scan_config($conf);
5295
5296     undef $skip_drive;
5297
5298     foreach my $snapname (keys %{$conf->{snapshots}}) {
5299         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5300     }
5301
5302     return 0;
5303 }
5304
5305 sub update_disksize {
5306     my ($vmid, $conf, $volid_hash) = @_;
5307
5308     my $changes;
5309
5310     my $used = {};
5311
5312     # Note: it is allowed to define multiple storages with same path (alias), so
5313     # we need to check both 'volid' and real 'path' (two different volid can point
5314     # to the same path).
5315
5316     my $usedpath = {};
5317
5318     # update size info
5319     foreach my $opt (keys %$conf) {
5320         if (is_valid_drivename($opt)) {
5321             my $drive = parse_drive($opt, $conf->{$opt});
5322             my $volid = $drive->{file};
5323             next if !$volid;
5324
5325             $used->{$volid} = 1;
5326             if ($volid_hash->{$volid} &&
5327                 (my $path = $volid_hash->{$volid}->{path})) {
5328                 $usedpath->{$path} = 1;
5329             }
5330
5331             next if drive_is_cdrom($drive);
5332             next if !$volid_hash->{$volid};
5333
5334             $drive->{size} = $volid_hash->{$volid}->{size};
5335             my $new = print_drive($vmid, $drive);
5336             if ($new ne $conf->{$opt}) {
5337                 $changes = 1;
5338                 $conf->{$opt} = $new;
5339             }
5340         }
5341     }
5342
5343     # remove 'unusedX' entry if volume is used
5344     foreach my $opt (keys %$conf) {
5345         next if $opt !~ m/^unused\d+$/;
5346         my $volid = $conf->{$opt};
5347         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5348         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5349             $changes = 1;
5350             delete $conf->{$opt};
5351         }
5352     }
5353
5354     foreach my $volid (sort keys %$volid_hash) {
5355         next if $volid =~ m/vm-$vmid-state-/;
5356         next if $used->{$volid};
5357         my $path = $volid_hash->{$volid}->{path};
5358         next if !$path; # just to be sure
5359         next if $usedpath->{$path};
5360         $changes = 1;
5361         PVE::QemuConfig->add_unused_volume($conf, $volid);
5362         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5363     }
5364
5365     return $changes;
5366 }
5367
5368 sub rescan {
5369     my ($vmid, $nolock) = @_;
5370
5371     my $cfg = PVE::Storage::config();
5372
5373     my $volid_hash = scan_volids($cfg, $vmid);
5374
5375     my $updatefn =  sub {
5376         my ($vmid) = @_;
5377
5378         my $conf = PVE::QemuConfig->load_config($vmid);
5379
5380         PVE::QemuConfig->check_lock($conf);
5381
5382         my $vm_volids = {};
5383         foreach my $volid (keys %$volid_hash) {
5384             my $info = $volid_hash->{$volid};
5385             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5386         }
5387
5388         my $changes = update_disksize($vmid, $conf, $vm_volids);
5389
5390         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5391     };
5392
5393     if (defined($vmid)) {
5394         if ($nolock) {
5395             &$updatefn($vmid);
5396         } else {
5397             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5398         }
5399     } else {
5400         my $vmlist = config_list();
5401         foreach my $vmid (keys %$vmlist) {
5402             if ($nolock) {
5403                 &$updatefn($vmid);
5404             } else {
5405                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5406             }
5407         }
5408     }
5409 }
5410
5411 sub restore_vma_archive {
5412     my ($archive, $vmid, $user, $opts, $comp) = @_;
5413
5414     my $input = $archive eq '-' ? "<&STDIN" : undef;
5415     my $readfrom = $archive;
5416
5417     my $uncomp = '';
5418     if ($comp) {
5419         $readfrom = '-';
5420         my $qarchive = PVE::Tools::shellquote($archive);
5421         if ($comp eq 'gzip') {
5422             $uncomp = "zcat $qarchive|";
5423         } elsif ($comp eq 'lzop') {
5424             $uncomp = "lzop -d -c $qarchive|";
5425         } else {
5426             die "unknown compression method '$comp'\n";
5427         }
5428
5429     }
5430
5431     my $tmpdir = "/var/tmp/vzdumptmp$$";
5432     rmtree $tmpdir;
5433
5434     # disable interrupts (always do cleanups)
5435     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5436         warn "got interrupt - ignored\n";
5437     };
5438
5439     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5440     POSIX::mkfifo($mapfifo, 0600);
5441     my $fifofh;
5442
5443     my $openfifo = sub {
5444         open($fifofh, '>', $mapfifo) || die $!;
5445     };
5446
5447     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5448
5449     my $oldtimeout;
5450     my $timeout = 5;
5451
5452     my $devinfo = {};
5453
5454     my $rpcenv = PVE::RPCEnvironment::get();
5455
5456     my $conffile = PVE::QemuConfig->config_file($vmid);
5457     my $tmpfn = "$conffile.$$.tmp";
5458
5459     # Note: $oldconf is undef if VM does not exists
5460     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5461     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5462
5463     my $print_devmap = sub {
5464         my $virtdev_hash = {};
5465
5466         my $cfgfn = "$tmpdir/qemu-server.conf";
5467
5468         # we can read the config - that is already extracted
5469         my $fh = IO::File->new($cfgfn, "r") ||
5470             "unable to read qemu-server.conf - $!\n";
5471
5472         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5473         if (-f $fwcfgfn) {
5474             my $pve_firewall_dir = '/etc/pve/firewall';
5475             mkdir $pve_firewall_dir; # make sure the dir exists
5476             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5477         }
5478
5479         while (defined(my $line = <$fh>)) {
5480             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5481                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5482                 die "archive does not contain data for drive '$virtdev'\n"
5483                     if !$devinfo->{$devname};
5484                 if (defined($opts->{storage})) {
5485                     $storeid = $opts->{storage} || 'local';
5486                 } elsif (!$storeid) {
5487                     $storeid = 'local';
5488                 }
5489                 $format = 'raw' if !$format;
5490                 $devinfo->{$devname}->{devname} = $devname;
5491                 $devinfo->{$devname}->{virtdev} = $virtdev;
5492                 $devinfo->{$devname}->{format} = $format;
5493                 $devinfo->{$devname}->{storeid} = $storeid;
5494
5495                 # check permission on storage
5496                 my $pool = $opts->{pool}; # todo: do we need that?
5497                 if ($user ne 'root@pam') {
5498                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5499                 }
5500
5501                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5502             }
5503         }
5504
5505         foreach my $devname (keys %$devinfo) {
5506             die "found no device mapping information for device '$devname'\n"
5507                 if !$devinfo->{$devname}->{virtdev};
5508         }
5509
5510         my $cfg = PVE::Storage::config();
5511
5512         # create empty/temp config
5513         if ($oldconf) {
5514             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5515             foreach_drive($oldconf, sub {
5516                 my ($ds, $drive) = @_;
5517
5518                 return if drive_is_cdrom($drive);
5519
5520                 my $volid = $drive->{file};
5521
5522                 return if !$volid || $volid =~ m|^/|;
5523
5524                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5525                 return if !$path || !$owner || ($owner != $vmid);
5526
5527                 # Note: only delete disk we want to restore
5528                 # other volumes will become unused
5529                 if ($virtdev_hash->{$ds}) {
5530                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5531                     if (my $err = $@) {
5532                         warn $err;
5533                     }
5534                 }
5535             });
5536
5537             # delete vmstate files
5538             # since after the restore we have no snapshots anymore
5539             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5540                 my $snap = $oldconf->{snapshots}->{$snapname};
5541                 if ($snap->{vmstate}) {
5542                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5543                     if (my $err = $@) {
5544                         warn $err;
5545                     }
5546                 }
5547             }
5548         }
5549
5550         my $map = {};
5551         foreach my $virtdev (sort keys %$virtdev_hash) {
5552             my $d = $virtdev_hash->{$virtdev};
5553             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5554             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5555
5556             # test if requested format is supported
5557             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5558             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5559             $d->{format} = $defFormat if !$supported;
5560
5561             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5562                                                   $d->{format}, undef, $alloc_size);
5563             print STDERR "new volume ID is '$volid'\n";
5564             $d->{volid} = $volid;
5565             my $path = PVE::Storage::path($cfg, $volid);
5566
5567             PVE::Storage::activate_volumes($cfg,[$volid]);
5568
5569             my $write_zeros = 1;
5570             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5571                 $write_zeros = 0;
5572             }
5573
5574             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5575
5576             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5577             $map->{$virtdev} = $volid;
5578         }
5579
5580         $fh->seek(0, 0) || die "seek failed - $!\n";
5581
5582         my $outfd = new IO::File ($tmpfn, "w") ||
5583             die "unable to write config for VM $vmid\n";
5584
5585         my $cookie = { netcount => 0 };
5586         while (defined(my $line = <$fh>)) {
5587             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5588         }
5589
5590         $fh->close();
5591         $outfd->close();
5592     };
5593
5594     eval {
5595         # enable interrupts
5596         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5597             die "interrupted by signal\n";
5598         };
5599         local $SIG{ALRM} = sub { die "got timeout\n"; };
5600
5601         $oldtimeout = alarm($timeout);
5602
5603         my $parser = sub {
5604             my $line = shift;
5605
5606             print "$line\n";
5607
5608             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5609                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5610                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5611             } elsif ($line =~ m/^CTIME: /) {
5612                 # we correctly received the vma config, so we can disable
5613                 # the timeout now for disk allocation (set to 10 minutes, so
5614                 # that we always timeout if something goes wrong)
5615                 alarm(600);
5616                 &$print_devmap();
5617                 print $fifofh "done\n";
5618                 my $tmp = $oldtimeout || 0;
5619                 $oldtimeout = undef;
5620                 alarm($tmp);
5621                 close($fifofh);
5622             }
5623         };
5624
5625         print "restore vma archive: $cmd\n";
5626         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5627     };
5628     my $err = $@;
5629
5630     alarm($oldtimeout) if $oldtimeout;
5631
5632     my $vollist = [];
5633     foreach my $devname (keys %$devinfo) {
5634         my $volid = $devinfo->{$devname}->{volid};
5635         push @$vollist, $volid if $volid;
5636     }
5637
5638     my $cfg = PVE::Storage::config();
5639     PVE::Storage::deactivate_volumes($cfg, $vollist);
5640
5641     unlink $mapfifo;
5642
5643     if ($err) {
5644         rmtree $tmpdir;
5645         unlink $tmpfn;
5646
5647         foreach my $devname (keys %$devinfo) {
5648             my $volid = $devinfo->{$devname}->{volid};
5649             next if !$volid;
5650             eval {
5651                 if ($volid =~ m|^/|) {
5652                     unlink $volid || die 'unlink failed\n';
5653                 } else {
5654                     PVE::Storage::vdisk_free($cfg, $volid);
5655                 }
5656                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5657             };
5658             print STDERR "unable to cleanup '$volid' - $@" if $@;
5659         }
5660         die $err;
5661     }
5662
5663     rmtree $tmpdir;
5664
5665     rename($tmpfn, $conffile) ||
5666         die "unable to commit configuration file '$conffile'\n";
5667
5668     PVE::Cluster::cfs_update(); # make sure we read new file
5669
5670     eval { rescan($vmid, 1); };
5671     warn $@ if $@;
5672 }
5673
5674 sub restore_tar_archive {
5675     my ($archive, $vmid, $user, $opts) = @_;
5676
5677     if ($archive ne '-') {
5678         my $firstfile = tar_archive_read_firstfile($archive);
5679         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5680             if $firstfile ne 'qemu-server.conf';
5681     }
5682
5683     my $storecfg = PVE::Storage::config();
5684
5685     # destroy existing data - keep empty config
5686     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5687     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5688
5689     my $tocmd = "/usr/lib/qemu-server/qmextract";
5690
5691     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5692     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5693     $tocmd .= ' --prealloc' if $opts->{prealloc};
5694     $tocmd .= ' --info' if $opts->{info};
5695
5696     # tar option "xf" does not autodetect compression when read from STDIN,
5697     # so we pipe to zcat
5698     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5699         PVE::Tools::shellquote("--to-command=$tocmd");
5700
5701     my $tmpdir = "/var/tmp/vzdumptmp$$";
5702     mkpath $tmpdir;
5703
5704     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5705     local $ENV{VZDUMP_VMID} = $vmid;
5706     local $ENV{VZDUMP_USER} = $user;
5707
5708     my $conffile = PVE::QemuConfig->config_file($vmid);
5709     my $tmpfn = "$conffile.$$.tmp";
5710
5711     # disable interrupts (always do cleanups)
5712     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5713         print STDERR "got interrupt - ignored\n";
5714     };
5715
5716     eval {
5717         # enable interrupts
5718         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5719             die "interrupted by signal\n";
5720         };
5721
5722         if ($archive eq '-') {
5723             print "extracting archive from STDIN\n";
5724             run_command($cmd, input => "<&STDIN");
5725         } else {
5726             print "extracting archive '$archive'\n";
5727             run_command($cmd);
5728         }
5729
5730         return if $opts->{info};
5731
5732         # read new mapping
5733         my $map = {};
5734         my $statfile = "$tmpdir/qmrestore.stat";
5735         if (my $fd = IO::File->new($statfile, "r")) {
5736             while (defined (my $line = <$fd>)) {
5737                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5738                     $map->{$1} = $2 if $1;
5739                 } else {
5740                     print STDERR "unable to parse line in statfile - $line\n";
5741                 }
5742             }
5743             $fd->close();
5744         }
5745
5746         my $confsrc = "$tmpdir/qemu-server.conf";
5747
5748         my $srcfd = new IO::File($confsrc, "r") ||
5749             die "unable to open file '$confsrc'\n";
5750
5751         my $outfd = new IO::File ($tmpfn, "w") ||
5752             die "unable to write config for VM $vmid\n";
5753
5754         my $cookie = { netcount => 0 };
5755         while (defined (my $line = <$srcfd>)) {
5756             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5757         }
5758
5759         $srcfd->close();
5760         $outfd->close();
5761     };
5762     my $err = $@;
5763
5764     if ($err) {
5765
5766         unlink $tmpfn;
5767
5768         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5769
5770         die $err;
5771     }
5772
5773     rmtree $tmpdir;
5774
5775     rename $tmpfn, $conffile ||
5776         die "unable to commit configuration file '$conffile'\n";
5777
5778     PVE::Cluster::cfs_update(); # make sure we read new file
5779
5780     eval { rescan($vmid, 1); };
5781     warn $@ if $@;
5782 };
5783
5784 sub foreach_writable_storage {
5785     my ($conf, $func) = @_;
5786
5787     my $sidhash = {};
5788
5789     foreach my $ds (keys %$conf) {
5790         next if !is_valid_drivename($ds);
5791
5792         my $drive = parse_drive($ds, $conf->{$ds});
5793         next if !$drive;
5794         next if drive_is_cdrom($drive);
5795
5796         my $volid = $drive->{file};
5797
5798         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5799         $sidhash->{$sid} = $sid if $sid;
5800     }
5801
5802     foreach my $sid (sort keys %$sidhash) {
5803         &$func($sid);
5804     }
5805 }
5806
5807 sub do_snapshots_with_qemu {
5808     my ($storecfg, $volid) = @_;
5809
5810     my $storage_name = PVE::Storage::parse_volume_id($volid);
5811
5812     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5813         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5814         return 1;
5815     }
5816
5817     if ($volid =~ m/\.(qcow2|qed)$/){
5818         return 1;
5819     }
5820
5821     return undef;
5822 }
5823
5824 sub qga_check_running {
5825     my ($vmid) = @_;
5826
5827     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5828     if ($@) {
5829         warn "Qemu Guest Agent are not running - $@";
5830         return 0;
5831     }
5832     return 1;
5833 }
5834
5835 sub template_create {
5836     my ($vmid, $conf, $disk) = @_;
5837
5838     my $storecfg = PVE::Storage::config();
5839
5840     foreach_drive($conf, sub {
5841         my ($ds, $drive) = @_;
5842
5843         return if drive_is_cdrom($drive);
5844         return if $disk && $ds ne $disk;
5845
5846         my $volid = $drive->{file};
5847         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5848
5849         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5850         $drive->{file} = $voliddst;
5851         $conf->{$ds} = print_drive($vmid, $drive);
5852         PVE::QemuConfig->write_config($vmid, $conf);
5853     });
5854 }
5855
5856 sub qemu_img_convert {
5857     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5858
5859     my $storecfg = PVE::Storage::config();
5860     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5861     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5862
5863     if ($src_storeid && $dst_storeid) {
5864
5865         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5866
5867         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5868         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5869
5870         my $src_format = qemu_img_format($src_scfg, $src_volname);
5871         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5872
5873         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5874         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5875
5876         my $cmd = [];
5877         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5878         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5879         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5880         if ($is_zero_initialized) {
5881             push @$cmd, "zeroinit:$dst_path";
5882         } else {
5883             push @$cmd, $dst_path;
5884         }
5885
5886         my $parser = sub {
5887             my $line = shift;
5888             if($line =~ m/\((\S+)\/100\%\)/){
5889                 my $percent = $1;
5890                 my $transferred = int($size * $percent / 100);
5891                 my $remaining = $size - $transferred;
5892
5893                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5894             }
5895
5896         };
5897
5898         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5899         my $err = $@;
5900         die "copy failed: $err" if $err;
5901     }
5902 }
5903
5904 sub qemu_img_format {
5905     my ($scfg, $volname) = @_;
5906
5907     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5908         return $1;
5909     } else {
5910         return "raw";
5911     }
5912 }
5913
5914 sub qemu_drive_mirror {
5915     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5916
5917     $jobs = {} if !$jobs;
5918
5919     my $qemu_target;
5920     my $format;
5921     $jobs->{"drive-$drive"} = {};
5922
5923     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5924         my $server = $1;
5925         my $port = $2;
5926         my $exportname = $3;
5927
5928         $format = "nbd";
5929         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5930         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5931         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5932
5933         my $pid = fork();
5934         if (!defined($pid)) {
5935             die "forking socat tunnel failed\n";
5936         } elsif ($pid == 0) {
5937             exec(@$cmd);
5938             warn "exec failed: $!\n";
5939             POSIX::_exit(-1);
5940         }
5941         $jobs->{"drive-$drive"}->{pid} = $pid;
5942
5943         my $timeout = 0;
5944         while (!-S $unixsocket) {
5945             die "nbd connection helper timed out\n"
5946                 if $timeout++ > 5;
5947             sleep 1;
5948         }
5949     } else {
5950         my $storecfg = PVE::Storage::config();
5951         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5952
5953         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5954
5955         $format = qemu_img_format($dst_scfg, $dst_volname);
5956
5957         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5958
5959         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5960     }
5961
5962     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5963     $opts->{format} = $format if $format;
5964
5965     print "drive mirror is starting for drive-$drive\n";
5966
5967     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5968
5969     if (my $err = $@) {
5970         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5971         die "mirroring error: $err";
5972     }
5973
5974     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5975 }
5976
5977 sub qemu_drive_mirror_monitor {
5978     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5979
5980     eval {
5981         my $err_complete = 0;
5982
5983         while (1) {
5984             die "storage migration timed out\n" if $err_complete > 300;
5985
5986             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5987
5988             my $running_mirror_jobs = {};
5989             foreach my $stat (@$stats) {
5990                 next if $stat->{type} ne 'mirror';
5991                 $running_mirror_jobs->{$stat->{device}} = $stat;
5992             }
5993
5994             my $readycounter = 0;
5995
5996             foreach my $job (keys %$jobs) {
5997
5998                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
5999                     print "$job : finished\n";
6000                     delete $jobs->{$job};
6001                     next;
6002                 }
6003
6004                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6005
6006                 my $busy = $running_mirror_jobs->{$job}->{busy};
6007                 my $ready = $running_mirror_jobs->{$job}->{ready};
6008                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6009                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6010                     my $remaining = $total - $transferred;
6011                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6012
6013                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6014                 }
6015
6016                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6017             }
6018
6019             last if scalar(keys %$jobs) == 0;
6020
6021             if ($readycounter == scalar(keys %$jobs)) {
6022                 print "all mirroring jobs are ready \n";
6023                 last if $skipcomplete; #do the complete later
6024
6025                 if ($vmiddst && $vmiddst != $vmid) {
6026                     if ($qga) {
6027                         print "freeze filesystem\n";
6028                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6029                     } else {
6030                         print "suspend vm\n";
6031                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6032                     }
6033
6034                     # if we clone a disk for a new target vm, we don't switch the disk
6035                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6036
6037                     if ($qga) {
6038                         print "unfreeze filesystem\n";
6039                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6040                     } else {
6041                         print "resume vm\n";
6042                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6043                     }
6044
6045                     last;
6046                 } else {
6047
6048                     foreach my $job (keys %$jobs) {
6049                         # try to switch the disk if source and destination are on the same guest
6050                         print "$job: Completing block job...\n";
6051
6052                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6053                         if ($@ =~ m/cannot be completed/) {
6054                             print "$job: Block job cannot be completed, try again.\n";
6055                             $err_complete++;
6056                         }else {
6057                             print "$job: Completed successfully.\n";
6058                             $jobs->{$job}->{complete} = 1;
6059                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6060                         }
6061                     }
6062                 }
6063             }
6064             sleep 1;
6065         }
6066     };
6067     my $err = $@;
6068
6069     if ($err) {
6070         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6071         die "mirroring error: $err";
6072     }
6073
6074 }
6075
6076 sub qemu_blockjobs_cancel {
6077     my ($vmid, $jobs) = @_;
6078
6079     foreach my $job (keys %$jobs) {
6080         print "$job: Cancelling block job\n";
6081         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6082         $jobs->{$job}->{cancel} = 1;
6083     }
6084
6085     while (1) {
6086         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6087
6088         my $running_jobs = {};
6089         foreach my $stat (@$stats) {
6090             $running_jobs->{$stat->{device}} = $stat;
6091         }
6092
6093         foreach my $job (keys %$jobs) {
6094
6095             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6096                 print "$job: Done.\n";
6097                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6098                 delete $jobs->{$job};
6099             }
6100         }
6101
6102         last if scalar(keys %$jobs) == 0;
6103
6104         sleep 1;
6105     }
6106 }
6107
6108 sub qemu_blockjobs_finish_tunnel {
6109    my ($vmid, $job, $cpid) = @_;
6110
6111    return if !$cpid;
6112
6113    for (my $i = 1; $i < 20; $i++) {
6114         my $waitpid = waitpid($cpid, WNOHANG);
6115         last if (defined($waitpid) && ($waitpid == $cpid));
6116
6117         if ($i == 10) {
6118             kill(15, $cpid);
6119         } elsif ($i >= 15) {
6120             kill(9, $cpid);
6121         }
6122         sleep (1);
6123     }
6124     unlink "/run/qemu-server/$vmid.mirror-$job";
6125 }
6126
6127 sub clone_disk {
6128     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6129         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6130
6131     my $newvolid;
6132
6133     if (!$full) {
6134         print "create linked clone of drive $drivename ($drive->{file})\n";
6135         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6136         push @$newvollist, $newvolid;
6137     } else {
6138
6139         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6140         $storeid = $storage if $storage;
6141
6142         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6143         if (!$format) {
6144             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6145             $format = qemu_img_format($scfg, $volname);
6146         }
6147
6148         # test if requested format is supported - else use default
6149         my $supported = grep { $_ eq $format } @$validFormats;
6150         $format = $defFormat if !$supported;
6151
6152         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6153
6154         print "create full clone of drive $drivename ($drive->{file})\n";
6155         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6156         push @$newvollist, $newvolid;
6157
6158         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6159
6160         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6161         if (!$running || $snapname) {
6162             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6163         } else {
6164
6165             my $kvmver = get_running_qemu_version ($vmid);
6166             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6167                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6168                     if $drive->{iothread};
6169             }
6170
6171             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6172         }
6173     }
6174
6175     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6176
6177     my $disk = $drive;
6178     $disk->{format} = undef;
6179     $disk->{file} = $newvolid;
6180     $disk->{size} = $size;
6181
6182     return $disk;
6183 }
6184
6185 # this only works if VM is running
6186 sub get_current_qemu_machine {
6187     my ($vmid) = @_;
6188
6189     my $cmd = { execute => 'query-machines', arguments => {} };
6190     my $res = vm_qmp_command($vmid, $cmd);
6191
6192     my ($current, $default);
6193     foreach my $e (@$res) {
6194         $default = $e->{name} if $e->{'is-default'};
6195         $current = $e->{name} if $e->{'is-current'};
6196     }
6197
6198     # fallback to the default machine if current is not supported by qemu
6199     return $current || $default || 'pc';
6200 }
6201
6202 sub get_running_qemu_version {
6203     my ($vmid) = @_;
6204     my $cmd = { execute => 'query-version', arguments => {} };
6205     my $res = vm_qmp_command($vmid, $cmd);
6206     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6207 }
6208
6209 sub qemu_machine_feature_enabled {
6210     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6211
6212     my $current_major;
6213     my $current_minor;
6214
6215     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6216
6217         $current_major = $3;
6218         $current_minor = $4;
6219
6220     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6221
6222         $current_major = $1;
6223         $current_minor = $2;
6224     }
6225
6226     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6227
6228
6229 }
6230
6231 sub qemu_machine_pxe {
6232     my ($vmid, $conf, $machine) = @_;
6233
6234     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6235
6236     foreach my $opt (keys %$conf) {
6237         next if $opt !~ m/^net(\d+)$/;
6238         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6239         next if !$net;
6240         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6241         return $machine.".pxe" if $romfile =~ m/pxe/;
6242         last;
6243     }
6244
6245     return $machine;
6246 }
6247
6248 sub qemu_use_old_bios_files {
6249     my ($machine_type) = @_;
6250
6251     return if !$machine_type;
6252
6253     my $use_old_bios_files = undef;
6254
6255     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6256         $machine_type = $1;
6257         $use_old_bios_files = 1;
6258     } else {
6259         my $kvmver = kvm_user_version();
6260         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6261         # load new efi bios files on migration. So this hack is required to allow
6262         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6263         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6264         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6265     }
6266
6267     return ($use_old_bios_files, $machine_type);
6268 }
6269
6270 sub lspci {
6271
6272     my $devices = {};
6273
6274     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6275             my (undef, $id, $function) = @_;
6276             my $res = { id => $id, function => $function};
6277             push @{$devices->{$id}}, $res;
6278     });
6279
6280     return $devices;
6281 }
6282
6283 sub vm_iothreads_list {
6284     my ($vmid) = @_;
6285
6286     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6287
6288     my $iothreads = {};
6289     foreach my $iothread (@$res) {
6290         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6291     }
6292
6293     return $iothreads;
6294 }
6295
6296 sub scsihw_infos {
6297     my ($conf, $drive) = @_;
6298
6299     my $maxdev = 0;
6300
6301     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6302         $maxdev = 7;
6303     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6304         $maxdev = 1;
6305     } else {
6306         $maxdev = 256;
6307     }
6308
6309     my $controller = int($drive->{index} / $maxdev);
6310     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6311
6312     return ($maxdev, $controller, $controller_prefix);
6313 }
6314
6315 sub add_hyperv_enlighments {
6316     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6317
6318     return if $nokvm;
6319     return if $winversion < 6;
6320     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6321
6322     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6323
6324     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6325         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6326         push @$cpuFlags , 'hv_vapic';
6327         push @$cpuFlags , 'hv_time';
6328     } else {
6329         push @$cpuFlags , 'hv_spinlocks=0xffff';
6330     }
6331
6332     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6333         push @$cpuFlags , 'hv_reset';
6334         push @$cpuFlags , 'hv_vpindex';
6335         push @$cpuFlags , 'hv_runtime';
6336     }
6337
6338     if ($winversion >= 7) {
6339         push @$cpuFlags , 'hv_relaxed';
6340     }
6341 }
6342
6343 sub windows_version {
6344     my ($ostype) = @_;
6345
6346     return 0 if !$ostype;
6347
6348     my $winversion = 0;
6349
6350     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6351         $winversion = 5;
6352     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6353         $winversion = 6;
6354     } elsif ($ostype =~ m/^win(\d+)$/) {
6355         $winversion = $1;
6356     }
6357
6358     return $winversion;
6359 }
6360
6361 # bash completion helper
6362
6363 sub complete_backup_archives {
6364     my ($cmdname, $pname, $cvalue) = @_;
6365
6366     my $cfg = PVE::Storage::config();
6367
6368     my $storeid;
6369
6370     if ($cvalue =~ m/^([^:]+):/) {
6371         $storeid = $1;
6372     }
6373
6374     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6375
6376     my $res = [];
6377     foreach my $id (keys %$data) {
6378         foreach my $item (@{$data->{$id}}) {
6379             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6380             push @$res, $item->{volid} if defined($item->{volid});
6381         }
6382     }
6383
6384     return $res;
6385 }
6386
6387 my $complete_vmid_full = sub {
6388     my ($running) = @_;
6389
6390     my $idlist = vmstatus();
6391
6392     my $res = [];
6393
6394     foreach my $id (keys %$idlist) {
6395         my $d = $idlist->{$id};
6396         if (defined($running)) {
6397             next if $d->{template};
6398             next if $running && $d->{status} ne 'running';
6399             next if !$running && $d->{status} eq 'running';
6400         }
6401         push @$res, $id;
6402
6403     }
6404     return $res;
6405 };
6406
6407 sub complete_vmid {
6408     return &$complete_vmid_full();
6409 }
6410
6411 sub complete_vmid_stopped {
6412     return &$complete_vmid_full(0);
6413 }
6414
6415 sub complete_vmid_running {
6416     return &$complete_vmid_full(1);
6417 }
6418
6419 sub complete_storage {
6420
6421     my $cfg = PVE::Storage::config();
6422     my $ids = $cfg->{ids};
6423
6424     my $res = [];
6425     foreach my $sid (keys %$ids) {
6426         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6427         next if !$ids->{$sid}->{content}->{images};
6428         push @$res, $sid;
6429     }
6430
6431     return $res;
6432 }
6433
6434 sub nbd_stop {
6435     my ($vmid) = @_;
6436
6437     vm_mon_cmd($vmid, 'nbd-server-stop');
6438 }
6439
6440 1;