PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     replica => {
 495         optional => 1,
 496         description => "Storage replica for local storage.",
 497         type => 'boolean',
 498         default => 0,
 499     },
 500     replica_rate_limit => {
 501         optional => 1,
 502         description => "Storage replica rate limit in KBytes/s.",
 503         type => 'integer',
 504         minimum => 1,
 505     },
 506     replica_interval => {
 507         optional => 1,
 508         description => "Storage replica sync interval",
 509         type => 'integer',
 510         minimum => 1,
 511         maximum => 1440,
 512         default => 15,
 513     },
 514     replica_target => {
 515         optional => 1,
 516         description => "Storage replica taget node.",
 517         type => 'string',
 518     },
 519     protection => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 523         default => 0,
 524     },
 525     bios => {
 526         optional => 1,
 527         type => 'string',
 528         enum => [ qw(seabios ovmf) ],
 529         description => "Select BIOS implementation.",
 530         default => 'seabios',
 531     },
 532 };
 533
 534 # what about other qemu settings ?
 535 #cpu => 'string',
 536 #machine => 'string',
 537 #fda => 'file',
 538 #fdb => 'file',
 539 #mtdblock => 'file',
 540 #sd => 'file',
 541 #pflash => 'file',
 542 #snapshot => 'bool',
 543 #bootp => 'file',
 544 ##tftp => 'dir',
 545 ##smb => 'dir',
 546 #kernel => 'file',
 547 #append => 'string',
 548 #initrd => 'file',
 549 ##soundhw => 'string',
 550
 551 while (my ($k, $v) = each %$confdesc) {
 552     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 553 }
 554
 555 my $MAX_IDE_DISKS = 4;
 556 my $MAX_SCSI_DISKS = 14;
 557 my $MAX_VIRTIO_DISKS = 16;
 558 my $MAX_SATA_DISKS = 6;
 559 my $MAX_USB_DEVICES = 5;
 560 my $MAX_NETS = 32;
 561 my $MAX_UNUSED_DISKS = 8;
 562 my $MAX_HOSTPCI_DEVICES = 4;
 563 my $MAX_SERIAL_PORTS = 4;
 564 my $MAX_PARALLEL_PORTS = 3;
 565 my $MAX_NUMA = 8;
 566
 567 my $numa_fmt = {
 568     cpus => {
 569         type => "string",
 570         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 571         description => "CPUs accessing this NUMA node.",
 572         format_description => "id[-id];...",
 573     },
 574     memory => {
 575         type => "number",
 576         description => "Amount of memory this NUMA node provides.",
 577         optional => 1,
 578     },
 579     hostnodes => {
 580         type => "string",
 581         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 582         description => "Host NUMA nodes to use.",
 583         format_description => "id[-id];...",
 584         optional => 1,
 585     },
 586     policy => {
 587         type => 'string',
 588         enum => [qw(preferred bind interleave)],
 589         description => "NUMA allocation policy.",
 590         optional => 1,
 591     },
 592 };
 593 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 594 my $numadesc = {
 595     optional => 1,
 596     type => 'string', format => $numa_fmt,
 597     description => "NUMA topology.",
 598 };
 599 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 600
 601 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 602     $confdesc->{"numa$i"} = $numadesc;
 603 }
 604
 605 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 606                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 607                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 608 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 609
 610 my $net_fmt_bridge_descr = <<__EOD__;
 611 Bridge to attach the network device to. The Proxmox VE standard bridge
 612 is called 'vmbr0'.
 613
 614 If you do not specify a bridge, we create a kvm user (NATed) network
 615 device, which provides DHCP and DNS services. The following addresses
 616 are used:
 617
 618  10.0.2.2   Gateway
 619  10.0.2.3   DNS Server
 620  10.0.2.4   SMB Server
 621
 622 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 623 __EOD__
 624
 625 my $net_fmt = {
 626     macaddr => {
 627         type => 'string',
 628         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 629         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 630         format_description => "XX:XX:XX:XX:XX:XX",
 631         optional => 1,
 632     },
 633     model => {
 634         type => 'string',
 635         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 636         enum => $nic_model_list,
 637         default_key => 1,
 638     },
 639     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 640     bridge => {
 641         type => 'string',
 642         description => $net_fmt_bridge_descr,
 643         format_description => 'bridge',
 644         optional => 1,
 645     },
 646     queues => {
 647         type => 'integer',
 648         minimum => 0, maximum => 16,
 649         description => 'Number of packet queues to be used on the device.',
 650         optional => 1,
 651     },
 652     rate => {
 653         type => 'number',
 654         minimum => 0,
 655         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 656         optional => 1,
 657     },
 658     tag => {
 659         type => 'integer',
 660         minimum => 1, maximum => 4094,
 661         description => 'VLAN tag to apply to packets on this interface.',
 662         optional => 1,
 663     },
 664     trunks => {
 665         type => 'string',
 666         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 667         description => 'VLAN trunks to pass through this interface.',
 668         format_description => 'vlanid[;vlanid...]',
 669         optional => 1,
 670     },
 671     firewall => {
 672         type => 'boolean',
 673         description => 'Whether this interface should be protected by the firewall.',
 674         optional => 1,
 675     },
 676     link_down => {
 677         type => 'boolean',
 678         description => 'Whether this interface should be disconnected (like pulling the plug).',
 679         optional => 1,
 680     },
 681 };
 682
 683 my $netdesc = {
 684     optional => 1,
 685     type => 'string', format => $net_fmt,
 686     description => "Specify network devices.",
 687 };
 688
 689 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 690
 691 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 692     $confdesc->{"net$i"} = $netdesc;
 693 }
 694
 695 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 696 sub verify_volume_id_or_qm_path {
 697     my ($volid, $noerr) = @_;
 698
 699     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 700         return $volid;
 701     }
 702
 703     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 704     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 705     if ($@) {
 706         return undef if $noerr;
 707         die $@;
 708     }
 709     return $volid;
 710 }
 711
 712 my $drivename_hash;
 713
 714 my %drivedesc_base = (
 715     volume => { alias => 'file' },
 716     file => {
 717         type => 'string',
 718         format => 'pve-volume-id-or-qm-path',
 719         default_key => 1,
 720         format_description => 'volume',
 721         description => "The drive's backing volume.",
 722     },
 723     media => {
 724         type => 'string',
 725         enum => [qw(cdrom disk)],
 726         description => "The drive's media type.",
 727         default => 'disk',
 728         optional => 1
 729     },
 730     cyls => {
 731         type => 'integer',
 732         description => "Force the drive's physical geometry to have a specific cylinder count.",
 733         optional => 1
 734     },
 735     heads => {
 736         type => 'integer',
 737         description => "Force the drive's physical geometry to have a specific head count.",
 738         optional => 1
 739     },
 740     secs => {
 741         type => 'integer',
 742         description => "Force the drive's physical geometry to have a specific sector count.",
 743         optional => 1
 744     },
 745     trans => {
 746         type => 'string',
 747         enum => [qw(none lba auto)],
 748         description => "Force disk geometry bios translation mode.",
 749         optional => 1,
 750     },
 751     snapshot => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making snapshots.",
 754         optional => 1,
 755     },
 756     cache => {
 757         type => 'string',
 758         enum => [qw(none writethrough writeback unsafe directsync)],
 759         description => "The drive's cache mode",
 760         optional => 1,
 761     },
 762     format => get_standard_option('pve-qm-image-format'),
 763     size => {
 764         type => 'string',
 765         format => 'disk-size',
 766         format_description => 'DiskSize',
 767         description => "Disk size. This is purely informational and has no effect.",
 768         optional => 1,
 769     },
 770     backup => {
 771         type => 'boolean',
 772         description => "Whether the drive should be included when making backups.",
 773         optional => 1,
 774     },
 775     replica => {
 776         type => 'boolean',
 777         description => 'Will include this drive to an storage replical job.',
 778         optional => 1,
 779         default => 1,
 780     },
 781     rerror => {
 782         type => 'string',
 783         enum => [qw(ignore report stop)],
 784         description => 'Read error action.',
 785         optional => 1,
 786     },
 787     werror => {
 788         type => 'string',
 789         enum => [qw(enospc ignore report stop)],
 790         description => 'Write error action.',
 791         optional => 1,
 792     },
 793     aio => {
 794         type => 'string',
 795         enum => [qw(native threads)],
 796         description => 'AIO type to use.',
 797         optional => 1,
 798     },
 799     discard => {
 800         type => 'string',
 801         enum => [qw(ignore on)],
 802         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 803         optional => 1,
 804     },
 805     detect_zeroes => {
 806         type => 'boolean',
 807         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 808         optional => 1,
 809     },
 810     serial => {
 811         type => 'string',
 812         format => 'urlencoded',
 813         format_description => 'serial',
 814         maxLength => 20*3, # *3 since it's %xx url enoded
 815         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 816         optional => 1,
 817     }
 818 );
 819
 820 my %iothread_fmt = ( iothread => {
 821         type => 'boolean',
 822         description => "Whether to use iothreads for this drive",
 823         optional => 1,
 824 });
 825
 826 my %model_fmt = (
 827     model => {
 828         type => 'string',
 829         format => 'urlencoded',
 830         format_description => 'model',
 831         maxLength => 40*3, # *3 since it's %xx url enoded
 832         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 833         optional => 1,
 834     },
 835 );
 836
 837 my %queues_fmt = (
 838     queues => {
 839         type => 'integer',
 840         description => "Number of queues.",
 841         minimum => 2,
 842         optional => 1
 843     }
 844 );
 845
 846 my %scsiblock_fmt = (
 847     scsiblock => {
 848         type => 'boolean',
 849         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 850         optional => 1,
 851         default => 0,
 852     },
 853 );
 854
 855 my $add_throttle_desc = sub {
 856     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 857     my $d = {
 858         type => $type,
 859         format_description => $unit,
 860         description => "Maximum $what in $longunit.",
 861         optional => 1,
 862     };
 863     $d->{minimum} = $minimum if defined($minimum);
 864     $drivedesc_base{$key} = $d;
 865 };
 866 # throughput: (leaky bucket)
 867 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 868 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 869 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 870 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 871 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 872 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 873 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 874 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 875 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 876
 877 # pools: (pool of IO before throttling starts taking effect)
 878 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 879 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 880 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 881 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 882 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 883 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 884
 885 # burst lengths
 886 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 887 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 888 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 889 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 890 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 891 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 892
 893 my $ide_fmt = {
 894     %drivedesc_base,
 895     %model_fmt,
 896 };
 897 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 898
 899 my $idedesc = {
 900     optional => 1,
 901     type => 'string', format => $ide_fmt,
 902     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 903 };
 904 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 905
 906 my $scsi_fmt = {
 907     %drivedesc_base,
 908     %iothread_fmt,
 909     %queues_fmt,
 910     %scsiblock_fmt,
 911 };
 912 my $scsidesc = {
 913     optional => 1,
 914     type => 'string', format => $scsi_fmt,
 915     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 916 };
 917 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 918
 919 my $sata_fmt = {
 920     %drivedesc_base,
 921 };
 922 my $satadesc = {
 923     optional => 1,
 924     type => 'string', format => $sata_fmt,
 925     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 926 };
 927 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 928
 929 my $virtio_fmt = {
 930     %drivedesc_base,
 931     %iothread_fmt,
 932 };
 933 my $virtiodesc = {
 934     optional => 1,
 935     type => 'string', format => $virtio_fmt,
 936     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 937 };
 938 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 939
 940 my $alldrive_fmt = {
 941     %drivedesc_base,
 942     %iothread_fmt,
 943     %model_fmt,
 944     %queues_fmt,
 945     %scsiblock_fmt,
 946 };
 947
 948 my $efidisk_fmt = {
 949     volume => { alias => 'file' },
 950     file => {
 951         type => 'string',
 952         format => 'pve-volume-id-or-qm-path',
 953         default_key => 1,
 954         format_description => 'volume',
 955         description => "The drive's backing volume.",
 956     },
 957     format => get_standard_option('pve-qm-image-format'),
 958     size => {
 959         type => 'string',
 960         format => 'disk-size',
 961         format_description => 'DiskSize',
 962         description => "Disk size. This is purely informational and has no effect.",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $efidisk_desc = {
 968     optional => 1,
 969     type => 'string', format => $efidisk_fmt,
 970     description => "Configure a Disk for storing EFI vars",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 974
 975 my $usb_fmt = {
 976     host => {
 977         default_key => 1,
 978         type => 'string', format => 'pve-qm-usb-device',
 979         format_description => 'HOSTUSBDEVICE|spice',
 980         description => <<EODESCR,
 981 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 982
 983  'bus-port(.port)*' (decimal numbers) or
 984  'vendor_id:product_id' (hexadeciaml numbers) or
 985  'spice'
 986
 987 You can use the 'lsusb -t' command to list existing usb devices.
 988
 989 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 990
 991 The value 'spice' can be used to add a usb redirection devices for spice.
 992 EODESCR
 993     },
 994     usb3 => {
 995         optional => 1,
 996         type => 'boolean',
 997         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 998         default => 0,
 999     },
1000 };
1001
1002 my $usbdesc = {
1003     optional => 1,
1004     type => 'string', format => $usb_fmt,
1005     description => "Configure an USB device (n is 0 to 4).",
1006 };
1007 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1008
1009 # NOTE: the match-groups of this regex are used in parse_hostpci
1010 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1011 my $hostpci_fmt = {
1012     host => {
1013         default_key => 1,
1014         type => 'string',
1015         pattern => qr/$PCIRE(;$PCIRE)*/,
1016         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1017         description => <<EODESCR,
1018 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1019 of PCI virtual functions of the host. HOSTPCIID syntax is:
1020
1021 'bus:dev.func' (hexadecimal numbers)
1022
1023 You can us the 'lspci' command to list existing PCI devices.
1024 EODESCR
1025     },
1026     rombar => {
1027         type => 'boolean',
1028         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1029         optional => 1,
1030         default => 1,
1031     },
1032     romfile => {
1033         type => 'string',
1034         pattern => '[^,;]+',
1035         format_description => 'string',
1036         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1037         optional => 1,
1038     },
1039     pcie => {
1040         type => 'boolean',
1041         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1042         optional => 1,
1043         default => 0,
1044     },
1045     'x-vga' => {
1046         type => 'boolean',
1047         description =>  "Enable vfio-vga device support.",
1048         optional => 1,
1049         default => 0,
1050     },
1051 };
1052 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1053
1054 my $hostpcidesc = {
1055         optional => 1,
1056         type => 'string', format => 'pve-qm-hostpci',
1057         description => "Map host PCI devices into guest.",
1058         verbose_description =>  <<EODESCR,
1059 Map host PCI devices into guest.
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer
1062 possible to migrate such machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1068
1069 my $serialdesc = {
1070         optional => 1,
1071         type => 'string',
1072         pattern => '(/dev/.+|socket)',
1073         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1074         verbose_description =>  <<EODESCR,
1075 Create a serial device inside the VM (n is 0 to 3), and pass through a
1076 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1077 host side (use 'qm terminal' to open a terminal connection).
1078
1079 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1080
1081 CAUTION: Experimental! User reported problems with this option.
1082 EODESCR
1083 };
1084
1085 my $paralleldesc= {
1086         optional => 1,
1087         type => 'string',
1088         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1089         description =>  "Map host parallel devices (n is 0 to 2).",
1090         verbose_description =>  <<EODESCR,
1091 Map host parallel devices (n is 0 to 2).
1092
1093 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1094
1095 CAUTION: Experimental! User reported problems with this option.
1096 EODESCR
1097 };
1098
1099 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1100     $confdesc->{"parallel$i"} = $paralleldesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1104     $confdesc->{"serial$i"} = $serialdesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1108     $confdesc->{"hostpci$i"} = $hostpcidesc;
1109 }
1110
1111 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1112     $drivename_hash->{"ide$i"} = 1;
1113     $confdesc->{"ide$i"} = $idedesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1117     $drivename_hash->{"sata$i"} = 1;
1118     $confdesc->{"sata$i"} = $satadesc;
1119 }
1120
1121 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1122     $drivename_hash->{"scsi$i"} = 1;
1123     $confdesc->{"scsi$i"} = $scsidesc ;
1124 }
1125
1126 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1127     $drivename_hash->{"virtio$i"} = 1;
1128     $confdesc->{"virtio$i"} = $virtiodesc;
1129 }
1130
1131 $drivename_hash->{efidisk0} = 1;
1132 $confdesc->{efidisk0} = $efidisk_desc;
1133
1134 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1135     $confdesc->{"usb$i"} = $usbdesc;
1136 }
1137
1138 my $unuseddesc = {
1139     optional => 1,
1140     type => 'string', format => 'pve-volume-id',
1141     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1142 };
1143
1144 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1145     $confdesc->{"unused$i"} = $unuseddesc;
1146 }
1147
1148 my $kvm_api_version = 0;
1149
1150 sub kvm_version {
1151
1152     return $kvm_api_version if $kvm_api_version;
1153
1154     my $fh = IO::File->new("</dev/kvm") ||
1155         return 0;
1156
1157     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1158         $kvm_api_version = $v;
1159     }
1160
1161     $fh->close();
1162
1163     return  $kvm_api_version;
1164 }
1165
1166 my $kvm_user_version;
1167
1168 sub kvm_user_version {
1169
1170     return $kvm_user_version if $kvm_user_version;
1171
1172     $kvm_user_version = 'unknown';
1173
1174     my $code = sub {
1175         my $line = shift;
1176         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1177             $kvm_user_version = $2;
1178         }
1179     };
1180
1181     eval { run_command("kvm -version", outfunc => $code); };
1182     warn $@ if $@;
1183
1184     return $kvm_user_version;
1185
1186 }
1187
1188 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1189
1190 sub valid_drive_names {
1191     # order is important - used to autoselect boot disk
1192     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1193             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1194             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1195             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1196             'efidisk0');
1197 }
1198
1199 sub is_valid_drivename {
1200     my $dev = shift;
1201
1202     return defined($drivename_hash->{$dev});
1203 }
1204
1205 sub option_exists {
1206     my $key = shift;
1207     return defined($confdesc->{$key});
1208 }
1209
1210 sub nic_models {
1211     return $nic_model_list;
1212 }
1213
1214 sub os_list_description {
1215
1216     return {
1217         other => 'Other',
1218         wxp => 'Windows XP',
1219         w2k => 'Windows 2000',
1220         w2k3 =>, 'Windows 2003',
1221         w2k8 => 'Windows 2008',
1222         wvista => 'Windows Vista',
1223         win7 => 'Windows 7',
1224         win8 => 'Windows 8/2012',
1225         win10 => 'Windows 10/2016',
1226         l24 => 'Linux 2.4',
1227         l26 => 'Linux 2.6',
1228     };
1229 }
1230
1231 my $cdrom_path;
1232
1233 sub get_cdrom_path {
1234
1235     return  $cdrom_path if $cdrom_path;
1236
1237     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1238     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1239     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1240 }
1241
1242 sub get_iso_path {
1243     my ($storecfg, $vmid, $cdrom) = @_;
1244
1245     if ($cdrom eq 'cdrom') {
1246         return get_cdrom_path();
1247     } elsif ($cdrom eq 'none') {
1248         return '';
1249     } elsif ($cdrom =~ m|^/|) {
1250         return $cdrom;
1251     } else {
1252         return PVE::Storage::path($storecfg, $cdrom);
1253     }
1254 }
1255
1256 # try to convert old style file names to volume IDs
1257 sub filename_to_volume_id {
1258     my ($vmid, $file, $media) = @_;
1259
1260     if (!($file eq 'none' || $file eq 'cdrom' ||
1261           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1262
1263         return undef if $file =~ m|/|;
1264
1265         if ($media && $media eq 'cdrom') {
1266             $file = "local:iso/$file";
1267         } else {
1268             $file = "local:$vmid/$file";
1269         }
1270     }
1271
1272     return $file;
1273 }
1274
1275 sub verify_media_type {
1276     my ($opt, $vtype, $media) = @_;
1277
1278     return if !$media;
1279
1280     my $etype;
1281     if ($media eq 'disk') {
1282         $etype = 'images';
1283     } elsif ($media eq 'cdrom') {
1284         $etype = 'iso';
1285     } else {
1286         die "internal error";
1287     }
1288
1289     return if ($vtype eq $etype);
1290
1291     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1292 }
1293
1294 sub cleanup_drive_path {
1295     my ($opt, $storecfg, $drive) = @_;
1296
1297     # try to convert filesystem paths to volume IDs
1298
1299     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1300         ($drive->{file} !~ m|^/dev/.+|) &&
1301         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1302         ($drive->{file} !~ m/^\d+$/)) {
1303         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1304         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1305         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1306         verify_media_type($opt, $vtype, $drive->{media});
1307         $drive->{file} = $volid;
1308     }
1309
1310     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1311 }
1312
1313 sub parse_hotplug_features {
1314     my ($data) = @_;
1315
1316     my $res = {};
1317
1318     return $res if $data eq '0';
1319
1320     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1321
1322     foreach my $feature (PVE::Tools::split_list($data)) {
1323         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1324             $res->{$1} = 1;
1325         } else {
1326             die "invalid hotplug feature '$feature'\n";
1327         }
1328     }
1329     return $res;
1330 }
1331
1332 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1333 sub pve_verify_hotplug_features {
1334     my ($value, $noerr) = @_;
1335
1336     return $value if parse_hotplug_features($value);
1337
1338     return undef if $noerr;
1339
1340     die "unable to parse hotplug option\n";
1341 }
1342
1343 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1344 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1345 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1346 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1347 #        [,iothread=on][,serial=serial][,model=model]
1348
1349 sub parse_drive {
1350     my ($key, $data) = @_;
1351
1352     my ($interface, $index);
1353
1354     if ($key =~ m/^([^\d]+)(\d+)$/) {
1355         $interface = $1;
1356         $index = $2;
1357     } else {
1358         return undef;
1359     }
1360
1361     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1362                                      : $confdesc->{$key}->{format};
1363     if (!$desc) {
1364         warn "invalid drive key: $key\n";
1365         return undef;
1366     }
1367     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1368     return undef if !$res;
1369     $res->{interface} = $interface;
1370     $res->{index} = $index;
1371
1372     my $error = 0;
1373     foreach my $opt (qw(bps bps_rd bps_wr)) {
1374         if (my $bps = defined(delete $res->{$opt})) {
1375             if (defined($res->{"m$opt"})) {
1376                 warn "both $opt and m$opt specified\n";
1377                 ++$error;
1378                 next;
1379             }
1380             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1381         }
1382     }
1383
1384     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1385     for my $requirement (
1386         [bps_max_length => 'mbps_max'],
1387         [bps_rd_max_length => 'mbps_rd_max'],
1388         [bps_wr_max_length => 'mbps_wr_max'],
1389         [iops_max_length => 'iops_max'],
1390         [iops_rd_max_length => 'iops_rd_max'],
1391         [iops_wr_max_length => 'iops_wr_max']) {
1392         my ($option, $requires) = @$requirement;
1393         if ($res->{$option} && !$res->{$requires}) {
1394             warn "$option requires $requires\n";
1395             ++$error;
1396         }
1397     }
1398
1399     return undef if $error;
1400
1401     return undef if $res->{mbps_rd} && $res->{mbps};
1402     return undef if $res->{mbps_wr} && $res->{mbps};
1403     return undef if $res->{iops_rd} && $res->{iops};
1404     return undef if $res->{iops_wr} && $res->{iops};
1405
1406     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1407         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1408         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1409         return undef if $res->{interface} eq 'virtio';
1410     }
1411
1412     if (my $size = $res->{size}) {
1413         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1414     }
1415
1416     return $res;
1417 }
1418
1419 sub print_drive {
1420     my ($vmid, $drive) = @_;
1421     my $data = { %$drive };
1422     delete $data->{$_} for qw(index interface);
1423     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1424 }
1425
1426 sub scsi_inquiry {
1427     my($fh, $noerr) = @_;
1428
1429     my $SG_IO = 0x2285;
1430     my $SG_GET_VERSION_NUM = 0x2282;
1431
1432     my $versionbuf = "\x00" x 8;
1433     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1434     if (!$ret) {
1435         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1436         return undef;
1437     }
1438     my $version = unpack("I", $versionbuf);
1439     if ($version < 30000) {
1440         die "scsi generic interface too old\n"  if !$noerr;
1441         return undef;
1442     }
1443
1444     my $buf = "\x00" x 36;
1445     my $sensebuf = "\x00" x 8;
1446     my $cmd = pack("C x3 C x1", 0x12, 36);
1447
1448     # see /usr/include/scsi/sg.h
1449     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1450
1451     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1452                       length($sensebuf), 0, length($buf), $buf,
1453                       $cmd, $sensebuf, 6000);
1454
1455     $ret = ioctl($fh, $SG_IO, $packet);
1456     if (!$ret) {
1457         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1458         return undef;
1459     }
1460
1461     my @res = unpack($sg_io_hdr_t, $packet);
1462     if ($res[17] || $res[18]) {
1463         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1464         return undef;
1465     }
1466
1467     my $res = {};
1468     (my $byte0, my $byte1, $res->{vendor},
1469      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1470
1471     $res->{removable} = $byte1 & 128 ? 1 : 0;
1472     $res->{type} = $byte0 & 31;
1473
1474     return $res;
1475 }
1476
1477 sub path_is_scsi {
1478     my ($path) = @_;
1479
1480     my $fh = IO::File->new("+<$path") || return undef;
1481     my $res = scsi_inquiry($fh, 1);
1482     close($fh);
1483
1484     return $res;
1485 }
1486
1487 sub machine_type_is_q35 {
1488     my ($conf) = @_;
1489
1490     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1491 }
1492
1493 sub print_tabletdevice_full {
1494     my ($conf) = @_;
1495
1496     my $q35 = machine_type_is_q35($conf);
1497
1498     # we use uhci for old VMs because tablet driver was buggy in older qemu
1499     my $usbbus = $q35 ? "ehci" : "uhci";
1500
1501     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1502 }
1503
1504 sub print_drivedevice_full {
1505     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1506
1507     my $device = '';
1508     my $maxdev = 0;
1509
1510     if ($drive->{interface} eq 'virtio') {
1511         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1512         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1513         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1514     } elsif ($drive->{interface} eq 'scsi') {
1515
1516         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1517         my $unit = $drive->{index} % $maxdev;
1518         my $devicetype = 'hd';
1519         my $path = '';
1520         if (drive_is_cdrom($drive)) {
1521             $devicetype = 'cd';
1522         } else {
1523             if ($drive->{file} =~ m|^/|) {
1524                 $path = $drive->{file};
1525                 if (my $info = path_is_scsi($path)) {
1526                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1527                         $devicetype = 'block';
1528                     } elsif ($info->{type} == 1) { # tape
1529                         $devicetype = 'generic';
1530                     }
1531                 }
1532             } else {
1533                  $path = PVE::Storage::path($storecfg, $drive->{file});
1534             }
1535
1536             if($path =~ m/^iscsi\:\/\//){
1537                 $devicetype = 'generic';
1538             }
1539         }
1540
1541         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1542             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1543         } else {
1544             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1545         }
1546
1547     } elsif ($drive->{interface} eq 'ide'){
1548         $maxdev = 2;
1549         my $controller = int($drive->{index} / $maxdev);
1550         my $unit = $drive->{index} % $maxdev;
1551         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1552
1553         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1554         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1555             $model = URI::Escape::uri_unescape($model);
1556             $device .= ",model=$model";
1557         }
1558     } elsif ($drive->{interface} eq 'sata'){
1559         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1560         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1561         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1562     } elsif ($drive->{interface} eq 'usb') {
1563         die "implement me";
1564         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1565     } else {
1566         die "unsupported interface type";
1567     }
1568
1569     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1570
1571     return $device;
1572 }
1573
1574 sub get_initiator_name {
1575     my $initiator;
1576
1577     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1578     while (defined(my $line = <$fh>)) {
1579         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1580         $initiator = $1;
1581         last;
1582     }
1583     $fh->close();
1584
1585     return $initiator;
1586 }
1587
1588 sub print_drive_full {
1589     my ($storecfg, $vmid, $drive) = @_;
1590
1591     my $path;
1592     my $volid = $drive->{file};
1593     my $format;
1594
1595     if (drive_is_cdrom($drive)) {
1596         $path = get_iso_path($storecfg, $vmid, $volid);
1597     } else {
1598         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1599         if ($storeid) {
1600             $path = PVE::Storage::path($storecfg, $volid);
1601             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1602             $format = qemu_img_format($scfg, $volname);
1603         } else {
1604             $path = $volid;
1605             $format = "raw";
1606         }
1607    }
1608
1609     my $opts = '';
1610     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1611     foreach my $o (@qemu_drive_options) {
1612         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1613     }
1614     if (my $serial = $drive->{serial}) {
1615         $serial = URI::Escape::uri_unescape($serial);
1616         $opts .= ",serial=$serial";
1617     }
1618
1619     $opts .= ",format=$format" if $format && !$drive->{format};
1620
1621     foreach my $o (qw(bps bps_rd bps_wr)) {
1622         my $v = $drive->{"m$o"};
1623         $opts .= ",$o=" . int($v*1024*1024) if $v;
1624     }
1625
1626     my $cache_direct = 0;
1627
1628     if (my $cache = $drive->{cache}) {
1629         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1630     } elsif (!drive_is_cdrom($drive)) {
1631         $opts .= ",cache=none";
1632         $cache_direct = 1;
1633     }
1634
1635     # aio native works only with O_DIRECT
1636     if (!$drive->{aio}) {
1637         if($cache_direct) {
1638             $opts .= ",aio=native";
1639         } else {
1640             $opts .= ",aio=threads";
1641         }
1642     }
1643
1644     if (!drive_is_cdrom($drive)) {
1645         my $detectzeroes;
1646         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1647             $detectzeroes = 'off';
1648         } elsif ($drive->{discard}) {
1649             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1650         } else {
1651             # This used to be our default with discard not being specified:
1652             $detectzeroes = 'on';
1653         }
1654         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1655     }
1656
1657     my $pathinfo = $path ? "file=$path," : '';
1658
1659     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1660 }
1661
1662 sub print_netdevice_full {
1663     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1664
1665     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1666
1667     my $device = $net->{model};
1668     if ($net->{model} eq 'virtio') {
1669          $device = 'virtio-net-pci';
1670      };
1671
1672     my $pciaddr = print_pci_addr("$netid", $bridges);
1673     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1674     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1675         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1676         my $vectors = $net->{queues} * 2 + 2;
1677         $tmpstr .= ",vectors=$vectors,mq=on";
1678     }
1679     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1680
1681     if ($use_old_bios_files) {
1682         my $romfile;
1683         if ($device eq 'virtio-net-pci') {
1684             $romfile = 'pxe-virtio.rom';
1685         } elsif ($device eq 'e1000') {
1686             $romfile = 'pxe-e1000.rom';
1687         } elsif ($device eq 'ne2k') {
1688             $romfile = 'pxe-ne2k_pci.rom';
1689         } elsif ($device eq 'pcnet') {
1690             $romfile = 'pxe-pcnet.rom';
1691         } elsif ($device eq 'rtl8139') {
1692             $romfile = 'pxe-rtl8139.rom';
1693         }
1694         $tmpstr .= ",romfile=$romfile" if $romfile;
1695     }
1696
1697     return $tmpstr;
1698 }
1699
1700 sub print_netdev_full {
1701     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1702
1703     my $i = '';
1704     if ($netid =~ m/^net(\d+)$/) {
1705         $i = int($1);
1706     }
1707
1708     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1709
1710     my $ifname = "tap${vmid}i$i";
1711
1712     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1713     die "interface name '$ifname' is too long (max 15 character)\n"
1714         if length($ifname) >= 16;
1715
1716     my $vhostparam = '';
1717     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1718
1719     my $vmname = $conf->{name} || "vm$vmid";
1720
1721     my $netdev = "";
1722     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1723
1724     if ($net->{bridge}) {
1725         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1726     } else {
1727         $netdev = "type=user,id=$netid,hostname=$vmname";
1728     }
1729
1730     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1731
1732     return $netdev;
1733 }
1734
1735
1736 sub print_cpu_device {
1737     my ($conf, $id) = @_;
1738
1739     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1740     my $cpu = $nokvm ? "qemu64" : "kvm64";
1741     if (my $cputype = $conf->{cpu}) {
1742         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1743             or die "Cannot parse cpu description: $cputype\n";
1744         $cpu = $cpuconf->{cputype};
1745     }
1746
1747     my $sockets = 1;
1748     $sockets = $conf->{sockets} if  $conf->{sockets};
1749     my $cores = $conf->{cores} || 1;
1750
1751     my $current_core = ($id - 1) % $cores;
1752     my $current_socket = int(($id - $current_core)/$cores);
1753
1754     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1755 }
1756
1757 sub drive_is_cdrom {
1758     my ($drive) = @_;
1759
1760     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1761
1762 }
1763
1764 sub parse_number_sets {
1765     my ($set) = @_;
1766     my $res = [];
1767     foreach my $part (split(/;/, $set)) {
1768         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1769             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1770             push @$res, [ $1, $2 ];
1771         } else {
1772             die "invalid range: $part\n";
1773         }
1774     }
1775     return $res;
1776 }
1777
1778 sub parse_numa {
1779     my ($data) = @_;
1780
1781     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1782     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1783     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1784     return $res;
1785 }
1786
1787 sub parse_hostpci {
1788     my ($value) = @_;
1789
1790     return undef if !$value;
1791
1792     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1793
1794     my @idlist = split(/;/, $res->{host});
1795     delete $res->{host};
1796     foreach my $id (@idlist) {
1797         if ($id =~ /^$PCIRE$/) {
1798             if (defined($2)) {
1799                 push @{$res->{pciid}}, { id => $1, function => $2 };
1800             } else {
1801                 my $pcidevices = lspci($1);
1802                 $res->{pciid} = $pcidevices->{$1};
1803             }
1804         } else {
1805             # should have been caught by parse_property_string already
1806             die "failed to parse PCI id: $id\n";
1807         }
1808     }
1809     return $res;
1810 }
1811
1812 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1813 sub parse_net {
1814     my ($data) = @_;
1815
1816     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1817     if ($@) {
1818         warn $@;
1819         return undef;
1820     }
1821     if (!defined($res->{macaddr})) {
1822         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1823         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1824     }
1825     return $res;
1826 }
1827
1828 sub print_net {
1829     my $net = shift;
1830
1831     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1832 }
1833
1834 sub add_random_macs {
1835     my ($settings) = @_;
1836
1837     foreach my $opt (keys %$settings) {
1838         next if $opt !~ m/^net(\d+)$/;
1839         my $net = parse_net($settings->{$opt});
1840         next if !$net;
1841         $settings->{$opt} = print_net($net);
1842     }
1843 }
1844
1845 sub vm_is_volid_owner {
1846     my ($storecfg, $vmid, $volid) = @_;
1847
1848     if ($volid !~  m|^/|) {
1849         my ($path, $owner);
1850         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1851         if ($owner && ($owner == $vmid)) {
1852             return 1;
1853         }
1854     }
1855
1856     return undef;
1857 }
1858
1859 sub split_flagged_list {
1860     my $text = shift || '';
1861     $text =~ s/[,;]/ /g;
1862     $text =~ s/^\s+//;
1863     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1864 }
1865
1866 sub join_flagged_list {
1867     my ($how, $lst) = @_;
1868     join $how, map { $lst->{$_} . $_ } keys %$lst;
1869 }
1870
1871 sub vmconfig_delete_pending_option {
1872     my ($conf, $key, $force) = @_;
1873
1874     delete $conf->{pending}->{$key};
1875     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1876     $pending_delete_hash->{$key} = $force ? '!' : '';
1877     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1878 }
1879
1880 sub vmconfig_undelete_pending_option {
1881     my ($conf, $key) = @_;
1882
1883     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1884     delete $pending_delete_hash->{$key};
1885
1886     if (%$pending_delete_hash) {
1887         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1888     } else {
1889         delete $conf->{pending}->{delete};
1890     }
1891 }
1892
1893 sub vmconfig_register_unused_drive {
1894     my ($storecfg, $vmid, $conf, $drive) = @_;
1895
1896     if (!drive_is_cdrom($drive)) {
1897         my $volid = $drive->{file};
1898         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1899             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1900         }
1901     }
1902 }
1903
1904 sub vmconfig_cleanup_pending {
1905     my ($conf) = @_;
1906
1907     # remove pending changes when nothing changed
1908     my $changes;
1909     foreach my $opt (keys %{$conf->{pending}}) {
1910         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1911             $changes = 1;
1912             delete $conf->{pending}->{$opt};
1913         }
1914     }
1915
1916     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1917     my $pending_delete_hash = {};
1918     while (my ($opt, $force) = each %$current_delete_hash) {
1919         if (defined($conf->{$opt})) {
1920             $pending_delete_hash->{$opt} = $force;
1921         } else {
1922             $changes = 1;
1923         }
1924     }
1925
1926     if (%$pending_delete_hash) {
1927         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1928     } else {
1929         delete $conf->{pending}->{delete};
1930     }
1931
1932     return $changes;
1933 }
1934
1935 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1936 my $smbios1_fmt = {
1937     uuid => {
1938         type => 'string',
1939         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1940         format_description => 'UUID',
1941         description => "Set SMBIOS1 UUID.",
1942         optional => 1,
1943     },
1944     version => {
1945         type => 'string',
1946         pattern => '\S+',
1947         format_description => 'string',
1948         description => "Set SMBIOS1 version.",
1949         optional => 1,
1950     },
1951     serial => {
1952         type => 'string',
1953         pattern => '\S+',
1954         format_description => 'string',
1955         description => "Set SMBIOS1 serial number.",
1956         optional => 1,
1957     },
1958     manufacturer => {
1959         type => 'string',
1960         pattern => '\S+',
1961         format_description => 'string',
1962         description => "Set SMBIOS1 manufacturer.",
1963         optional => 1,
1964     },
1965     product => {
1966         type => 'string',
1967         pattern => '\S+',
1968         format_description => 'string',
1969         description => "Set SMBIOS1 product ID.",
1970         optional => 1,
1971     },
1972     sku => {
1973         type => 'string',
1974         pattern => '\S+',
1975         format_description => 'string',
1976         description => "Set SMBIOS1 SKU string.",
1977         optional => 1,
1978     },
1979     family => {
1980         type => 'string',
1981         pattern => '\S+',
1982         format_description => 'string',
1983         description => "Set SMBIOS1 family string.",
1984         optional => 1,
1985     },
1986 };
1987
1988 sub parse_smbios1 {
1989     my ($data) = @_;
1990
1991     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1992     warn $@ if $@;
1993     return $res;
1994 }
1995
1996 sub print_smbios1 {
1997     my ($smbios1) = @_;
1998     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1999 }
2000
2001 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2002
2003 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2004 sub verify_bootdisk {
2005     my ($value, $noerr) = @_;
2006
2007     return $value if is_valid_drivename($value);
2008
2009     return undef if $noerr;
2010
2011     die "invalid boot disk '$value'\n";
2012 }
2013
2014 sub parse_watchdog {
2015     my ($value) = @_;
2016
2017     return undef if !$value;
2018
2019     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2020     warn $@ if $@;
2021     return $res;
2022 }
2023
2024 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2025 sub verify_usb_device {
2026     my ($value, $noerr) = @_;
2027
2028     return $value if parse_usb_device($value);
2029
2030     return undef if $noerr;
2031
2032     die "unable to parse usb device\n";
2033 }
2034
2035 # add JSON properties for create and set function
2036 sub json_config_properties {
2037     my $prop = shift;
2038
2039     foreach my $opt (keys %$confdesc) {
2040         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2041         $prop->{$opt} = $confdesc->{$opt};
2042     }
2043
2044     return $prop;
2045 }
2046
2047 sub check_type {
2048     my ($key, $value) = @_;
2049
2050     die "unknown setting '$key'\n" if !$confdesc->{$key};
2051
2052     my $type = $confdesc->{$key}->{type};
2053
2054     if (!defined($value)) {
2055         die "got undefined value\n";
2056     }
2057
2058     if ($value =~ m/[\n\r]/) {
2059         die "property contains a line feed\n";
2060     }
2061
2062     if ($type eq 'boolean') {
2063         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2064         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2065         die "type check ('boolean') failed - got '$value'\n";
2066     } elsif ($type eq 'integer') {
2067         return int($1) if $value =~ m/^(\d+)$/;
2068         die "type check ('integer') failed - got '$value'\n";
2069     } elsif ($type eq 'number') {
2070         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2071         die "type check ('number') failed - got '$value'\n";
2072     } elsif ($type eq 'string') {
2073         if (my $fmt = $confdesc->{$key}->{format}) {
2074             PVE::JSONSchema::check_format($fmt, $value);
2075             return $value;
2076         }
2077         $value =~ s/^\"(.*)\"$/$1/;
2078         return $value;
2079     } else {
2080         die "internal error"
2081     }
2082 }
2083
2084 sub check_iommu_support{
2085     #fixme : need to check IOMMU support
2086     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2087
2088     my $iommu=1;
2089     return $iommu;
2090
2091 }
2092
2093 sub touch_config {
2094     my ($vmid) = @_;
2095
2096     my $conf = PVE::QemuConfig->config_file($vmid);
2097     utime undef, undef, $conf;
2098 }
2099
2100 sub destroy_vm {
2101     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2102
2103     my $conffile = PVE::QemuConfig->config_file($vmid);
2104
2105     my $conf = PVE::QemuConfig->load_config($vmid);
2106
2107     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2108
2109     # only remove disks owned by this VM
2110     foreach_drive($conf, sub {
2111         my ($ds, $drive) = @_;
2112
2113         return if drive_is_cdrom($drive);
2114
2115         my $volid = $drive->{file};
2116
2117         return if !$volid || $volid =~ m|^/|;
2118
2119         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2120         return if !$path || !$owner || ($owner != $vmid);
2121
2122         eval {
2123             PVE::Storage::vdisk_free($storecfg, $volid);
2124         };
2125         warn "Could not remove disk '$volid', check manually: $@" if $@;
2126
2127     });
2128
2129     if ($keep_empty_config) {
2130         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2131     } else {
2132         unlink $conffile;
2133     }
2134
2135     # also remove unused disk
2136     eval {
2137         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2138
2139         eval {
2140             PVE::Storage::foreach_volid($dl, sub {
2141                 my ($volid, $sid, $volname, $d) = @_;
2142                 PVE::Storage::vdisk_free($storecfg, $volid);
2143             });
2144         };
2145         warn $@ if $@;
2146
2147     };
2148     warn $@ if $@;
2149 }
2150
2151 sub parse_vm_config {
2152     my ($filename, $raw) = @_;
2153
2154     return undef if !defined($raw);
2155
2156     my $res = {
2157         digest => Digest::SHA::sha1_hex($raw),
2158         snapshots => {},
2159         pending => {},
2160     };
2161
2162     $filename =~ m|/qemu-server/(\d+)\.conf$|
2163         || die "got strange filename '$filename'";
2164
2165     my $vmid = $1;
2166
2167     my $conf = $res;
2168     my $descr;
2169     my $section = '';
2170
2171     my @lines = split(/\n/, $raw);
2172     foreach my $line (@lines) {
2173         next if $line =~ m/^\s*$/;
2174
2175         if ($line =~ m/^\[PENDING\]\s*$/i) {
2176             $section = 'pending';
2177             if (defined($descr)) {
2178                 $descr =~ s/\s+$//;
2179                 $conf->{description} = $descr;
2180             }
2181             $descr = undef;
2182             $conf = $res->{$section} = {};
2183             next;
2184
2185         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2186             $section = $1;
2187             if (defined($descr)) {
2188                 $descr =~ s/\s+$//;
2189                 $conf->{description} = $descr;
2190             }
2191             $descr = undef;
2192             $conf = $res->{snapshots}->{$section} = {};
2193             next;
2194         }
2195
2196         if ($line =~ m/^\#(.*)\s*$/) {
2197             $descr = '' if !defined($descr);
2198             $descr .= PVE::Tools::decode_text($1) . "\n";
2199             next;
2200         }
2201
2202         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2203             $descr = '' if !defined($descr);
2204             $descr .= PVE::Tools::decode_text($2);
2205         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2206             $conf->{snapstate} = $1;
2207         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2208             my $key = $1;
2209             my $value = $2;
2210             $conf->{$key} = $value;
2211         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2212             my $value = $1;
2213             if ($section eq 'pending') {
2214                 $conf->{delete} = $value; # we parse this later
2215             } else {
2216                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2217             }
2218         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2219             my $key = $1;
2220             my $value = $2;
2221             eval { $value = check_type($key, $value); };
2222             if ($@) {
2223                 warn "vm $vmid - unable to parse value of '$key' - $@";
2224             } else {
2225                 $key = 'ide2' if $key eq 'cdrom';
2226                 my $fmt = $confdesc->{$key}->{format};
2227                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2228                     my $v = parse_drive($key, $value);
2229                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2230                         $v->{file} = $volid;
2231                         $value = print_drive($vmid, $v);
2232                     } else {
2233                         warn "vm $vmid - unable to parse value of '$key'\n";
2234                         next;
2235                     }
2236                 }
2237
2238                 $conf->{$key} = $value;
2239             }
2240         }
2241     }
2242
2243     if (defined($descr)) {
2244         $descr =~ s/\s+$//;
2245         $conf->{description} = $descr;
2246     }
2247     delete $res->{snapstate}; # just to be sure
2248
2249     return $res;
2250 }
2251
2252 sub write_vm_config {
2253     my ($filename, $conf) = @_;
2254
2255     delete $conf->{snapstate}; # just to be sure
2256
2257     if ($conf->{cdrom}) {
2258         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2259         $conf->{ide2} = $conf->{cdrom};
2260         delete $conf->{cdrom};
2261     }
2262
2263     # we do not use 'smp' any longer
2264     if ($conf->{sockets}) {
2265         delete $conf->{smp};
2266     } elsif ($conf->{smp}) {
2267         $conf->{sockets} = $conf->{smp};
2268         delete $conf->{cores};
2269         delete $conf->{smp};
2270     }
2271
2272     my $used_volids = {};
2273
2274     my $cleanup_config = sub {
2275         my ($cref, $pending, $snapname) = @_;
2276
2277         foreach my $key (keys %$cref) {
2278             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2279                 $key eq 'snapstate' || $key eq 'pending';
2280             my $value = $cref->{$key};
2281             if ($key eq 'delete') {
2282                 die "propertry 'delete' is only allowed in [PENDING]\n"
2283                     if !$pending;
2284                 # fixme: check syntax?
2285                 next;
2286             }
2287             eval { $value = check_type($key, $value); };
2288             die "unable to parse value of '$key' - $@" if $@;
2289
2290             $cref->{$key} = $value;
2291
2292             if (!$snapname && is_valid_drivename($key)) {
2293                 my $drive = parse_drive($key, $value);
2294                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2295             }
2296         }
2297     };
2298
2299     &$cleanup_config($conf);
2300
2301     &$cleanup_config($conf->{pending}, 1);
2302
2303     foreach my $snapname (keys %{$conf->{snapshots}}) {
2304         die "internal error" if $snapname eq 'pending';
2305         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2306     }
2307
2308     # remove 'unusedX' settings if we re-add a volume
2309     foreach my $key (keys %$conf) {
2310         my $value = $conf->{$key};
2311         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2312             delete $conf->{$key};
2313         }
2314     }
2315
2316     my $generate_raw_config = sub {
2317         my ($conf, $pending) = @_;
2318
2319         my $raw = '';
2320
2321         # add description as comment to top of file
2322         if (defined(my $descr = $conf->{description})) {
2323             if ($descr) {
2324                 foreach my $cl (split(/\n/, $descr)) {
2325                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2326                 }
2327             } else {
2328                 $raw .= "#\n" if $pending;
2329             }
2330         }
2331
2332         foreach my $key (sort keys %$conf) {
2333             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2334             $raw .= "$key: $conf->{$key}\n";
2335         }
2336         return $raw;
2337     };
2338
2339     my $raw = &$generate_raw_config($conf);
2340
2341     if (scalar(keys %{$conf->{pending}})){
2342         $raw .= "\n[PENDING]\n";
2343         $raw .= &$generate_raw_config($conf->{pending}, 1);
2344     }
2345
2346     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2347         $raw .= "\n[$snapname]\n";
2348         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2349     }
2350
2351     return $raw;
2352 }
2353
2354 sub load_defaults {
2355
2356     my $res = {};
2357
2358     # we use static defaults from our JSON schema configuration
2359     foreach my $key (keys %$confdesc) {
2360         if (defined(my $default = $confdesc->{$key}->{default})) {
2361             $res->{$key} = $default;
2362         }
2363     }
2364
2365     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2366     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2367
2368     return $res;
2369 }
2370
2371 sub config_list {
2372     my $vmlist = PVE::Cluster::get_vmlist();
2373     my $res = {};
2374     return $res if !$vmlist || !$vmlist->{ids};
2375     my $ids = $vmlist->{ids};
2376
2377     foreach my $vmid (keys %$ids) {
2378         my $d = $ids->{$vmid};
2379         next if !$d->{node} || $d->{node} ne $nodename;
2380         next if !$d->{type} || $d->{type} ne 'qemu';
2381         $res->{$vmid}->{exists} = 1;
2382     }
2383     return $res;
2384 }
2385
2386 # test if VM uses local resources (to prevent migration)
2387 sub check_local_resources {
2388     my ($conf, $noerr) = @_;
2389
2390     my $loc_res = 0;
2391
2392     $loc_res = 1 if $conf->{hostusb}; # old syntax
2393     $loc_res = 1 if $conf->{hostpci}; # old syntax
2394
2395     foreach my $k (keys %$conf) {
2396         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2397         # sockets are safe: they will recreated be on the target side post-migrate
2398         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2399         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2400     }
2401
2402     die "VM uses local resources\n" if $loc_res && !$noerr;
2403
2404     return $loc_res;
2405 }
2406
2407 # check if used storages are available on all nodes (use by migrate)
2408 sub check_storage_availability {
2409     my ($storecfg, $conf, $node) = @_;
2410
2411     foreach_drive($conf, sub {
2412         my ($ds, $drive) = @_;
2413
2414         my $volid = $drive->{file};
2415         return if !$volid;
2416
2417         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2418         return if !$sid;
2419
2420         # check if storage is available on both nodes
2421         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2422         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2423    });
2424 }
2425
2426 # list nodes where all VM images are available (used by has_feature API)
2427 sub shared_nodes {
2428     my ($conf, $storecfg) = @_;
2429
2430     my $nodelist = PVE::Cluster::get_nodelist();
2431     my $nodehash = { map { $_ => 1 } @$nodelist };
2432     my $nodename = PVE::INotify::nodename();
2433
2434     foreach_drive($conf, sub {
2435         my ($ds, $drive) = @_;
2436
2437         my $volid = $drive->{file};
2438         return if !$volid;
2439
2440         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2441         if ($storeid) {
2442             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2443             if ($scfg->{disable}) {
2444                 $nodehash = {};
2445             } elsif (my $avail = $scfg->{nodes}) {
2446                 foreach my $node (keys %$nodehash) {
2447                     delete $nodehash->{$node} if !$avail->{$node};
2448                 }
2449             } elsif (!$scfg->{shared}) {
2450                 foreach my $node (keys %$nodehash) {
2451                     delete $nodehash->{$node} if $node ne $nodename
2452                 }
2453             }
2454         }
2455     });
2456
2457     return $nodehash
2458 }
2459
2460 sub check_cmdline {
2461     my ($pidfile, $pid) = @_;
2462
2463     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2464     if (defined($fh)) {
2465         my $line = <$fh>;
2466         $fh->close;
2467         return undef if !$line;
2468         my @param = split(/\0/, $line);
2469
2470         my $cmd = $param[0];
2471         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2472
2473         for (my $i = 0; $i < scalar (@param); $i++) {
2474             my $p = $param[$i];
2475             next if !$p;
2476             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2477                 my $p = $param[$i+1];
2478                 return 1 if $p && ($p eq $pidfile);
2479                 return undef;
2480             }
2481         }
2482     }
2483     return undef;
2484 }
2485
2486 sub check_running {
2487     my ($vmid, $nocheck, $node) = @_;
2488
2489     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2490
2491     die "unable to find configuration file for VM $vmid - no such machine\n"
2492         if !$nocheck && ! -f $filename;
2493
2494     my $pidfile = pidfile_name($vmid);
2495
2496     if (my $fd = IO::File->new("<$pidfile")) {
2497         my $st = stat($fd);
2498         my $line = <$fd>;
2499         close($fd);
2500
2501         my $mtime = $st->mtime;
2502         if ($mtime > time()) {
2503             warn "file '$filename' modified in future\n";
2504         }
2505
2506         if ($line =~ m/^(\d+)$/) {
2507             my $pid = $1;
2508             if (check_cmdline($pidfile, $pid)) {
2509                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2510                     return $pid;
2511                 }
2512             }
2513         }
2514     }
2515
2516     return undef;
2517 }
2518
2519 sub vzlist {
2520
2521     my $vzlist = config_list();
2522
2523     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2524
2525     while (defined(my $de = $fd->read)) {
2526         next if $de !~ m/^(\d+)\.pid$/;
2527         my $vmid = $1;
2528         next if !defined($vzlist->{$vmid});
2529         if (my $pid = check_running($vmid)) {
2530             $vzlist->{$vmid}->{pid} = $pid;
2531         }
2532     }
2533
2534     return $vzlist;
2535 }
2536
2537 sub disksize {
2538     my ($storecfg, $conf) = @_;
2539
2540     my $bootdisk = $conf->{bootdisk};
2541     return undef if !$bootdisk;
2542     return undef if !is_valid_drivename($bootdisk);
2543
2544     return undef if !$conf->{$bootdisk};
2545
2546     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2547     return undef if !defined($drive);
2548
2549     return undef if drive_is_cdrom($drive);
2550
2551     my $volid = $drive->{file};
2552     return undef if !$volid;
2553
2554     return $drive->{size};
2555 }
2556
2557 my $last_proc_pid_stat;
2558
2559 # get VM status information
2560 # This must be fast and should not block ($full == false)
2561 # We only query KVM using QMP if $full == true (this can be slow)
2562 sub vmstatus {
2563     my ($opt_vmid, $full) = @_;
2564
2565     my $res = {};
2566
2567     my $storecfg = PVE::Storage::config();
2568
2569     my $list = vzlist();
2570     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2571
2572     my $cpucount = $cpuinfo->{cpus} || 1;
2573
2574     foreach my $vmid (keys %$list) {
2575         next if $opt_vmid && ($vmid ne $opt_vmid);
2576
2577         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2578         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2579
2580         my $d = {};
2581         $d->{pid} = $list->{$vmid}->{pid};
2582
2583         # fixme: better status?
2584         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2585
2586         my $size = disksize($storecfg, $conf);
2587         if (defined($size)) {
2588             $d->{disk} = 0; # no info available
2589             $d->{maxdisk} = $size;
2590         } else {
2591             $d->{disk} = 0;
2592             $d->{maxdisk} = 0;
2593         }
2594
2595         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2596         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2597         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2598
2599         $d->{name} = $conf->{name} || "VM $vmid";
2600         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2601
2602         if ($conf->{balloon}) {
2603             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2604             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2605         }
2606
2607         $d->{uptime} = 0;
2608         $d->{cpu} = 0;
2609         $d->{mem} = 0;
2610
2611         $d->{netout} = 0;
2612         $d->{netin} = 0;
2613
2614         $d->{diskread} = 0;
2615         $d->{diskwrite} = 0;
2616
2617         $d->{template} = PVE::QemuConfig->is_template($conf);
2618
2619         $res->{$vmid} = $d;
2620     }
2621
2622     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2623     foreach my $dev (keys %$netdev) {
2624         next if $dev !~ m/^tap([1-9]\d*)i/;
2625         my $vmid = $1;
2626         my $d = $res->{$vmid};
2627         next if !$d;
2628
2629         $d->{netout} += $netdev->{$dev}->{receive};
2630         $d->{netin} += $netdev->{$dev}->{transmit};
2631
2632         if ($full) {
2633             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2634             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2635         }
2636
2637     }
2638
2639     my $ctime = gettimeofday;
2640
2641     foreach my $vmid (keys %$list) {
2642
2643         my $d = $res->{$vmid};
2644         my $pid = $d->{pid};
2645         next if !$pid;
2646
2647         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2648         next if !$pstat; # not running
2649
2650         my $used = $pstat->{utime} + $pstat->{stime};
2651
2652         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2653
2654         if ($pstat->{vsize}) {
2655             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2656         }
2657
2658         my $old = $last_proc_pid_stat->{$pid};
2659         if (!$old) {
2660             $last_proc_pid_stat->{$pid} = {
2661                 time => $ctime,
2662                 used => $used,
2663                 cpu => 0,
2664             };
2665             next;
2666         }
2667
2668         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2669
2670         if ($dtime > 1000) {
2671             my $dutime = $used -  $old->{used};
2672
2673             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2674             $last_proc_pid_stat->{$pid} = {
2675                 time => $ctime,
2676                 used => $used,
2677                 cpu => $d->{cpu},
2678             };
2679         } else {
2680             $d->{cpu} = $old->{cpu};
2681         }
2682     }
2683
2684     return $res if !$full;
2685
2686     my $qmpclient = PVE::QMPClient->new();
2687
2688     my $ballooncb = sub {
2689         my ($vmid, $resp) = @_;
2690
2691         my $info = $resp->{'return'};
2692         return if !$info->{max_mem};
2693
2694         my $d = $res->{$vmid};
2695
2696         # use memory assigned to VM
2697         $d->{maxmem} = $info->{max_mem};
2698         $d->{balloon} = $info->{actual};
2699
2700         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2701             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2702             $d->{freemem} = $info->{free_mem};
2703         }
2704
2705         $d->{ballooninfo} = $info;
2706     };
2707
2708     my $blockstatscb = sub {
2709         my ($vmid, $resp) = @_;
2710         my $data = $resp->{'return'} || [];
2711         my $totalrdbytes = 0;
2712         my $totalwrbytes = 0;
2713
2714         for my $blockstat (@$data) {
2715             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2716             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2717
2718             $blockstat->{device} =~ s/drive-//;
2719             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2720         }
2721         $res->{$vmid}->{diskread} = $totalrdbytes;
2722         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2723     };
2724
2725     my $statuscb = sub {
2726         my ($vmid, $resp) = @_;
2727
2728         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2729         # this fails if ballon driver is not loaded, so this must be
2730         # the last commnand (following command are aborted if this fails).
2731         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2732
2733         my $status = 'unknown';
2734         if (!defined($status = $resp->{'return'}->{status})) {
2735             warn "unable to get VM status\n";
2736             return;
2737         }
2738
2739         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2740     };
2741
2742     foreach my $vmid (keys %$list) {
2743         next if $opt_vmid && ($vmid ne $opt_vmid);
2744         next if !$res->{$vmid}->{pid}; # not running
2745         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2746     }
2747
2748     $qmpclient->queue_execute(undef, 2);
2749
2750     foreach my $vmid (keys %$list) {
2751         next if $opt_vmid && ($vmid ne $opt_vmid);
2752         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2753     }
2754
2755     return $res;
2756 }
2757
2758 sub foreach_drive {
2759     my ($conf, $func, @param) = @_;
2760
2761     foreach my $ds (valid_drive_names()) {
2762         next if !defined($conf->{$ds});
2763
2764         my $drive = parse_drive($ds, $conf->{$ds});
2765         next if !$drive;
2766
2767         &$func($ds, $drive, @param);
2768     }
2769 }
2770
2771 sub foreach_volid {
2772     my ($conf, $func, @param) = @_;
2773
2774     my $volhash = {};
2775
2776     my $test_volid = sub {
2777         my ($volid, $is_cdrom) = @_;
2778
2779         return if !$volid;
2780
2781         $volhash->{$volid} = $is_cdrom || 0;
2782     };
2783
2784     foreach_drive($conf, sub {
2785         my ($ds, $drive) = @_;
2786         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2787     });
2788
2789     foreach my $snapname (keys %{$conf->{snapshots}}) {
2790         my $snap = $conf->{snapshots}->{$snapname};
2791         &$test_volid($snap->{vmstate}, 0);
2792         foreach_drive($snap, sub {
2793             my ($ds, $drive) = @_;
2794             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2795         });
2796     }
2797
2798     foreach my $volid (keys %$volhash) {
2799         &$func($volid, $volhash->{$volid}, @param);
2800     }
2801 }
2802
2803 sub vga_conf_has_spice {
2804     my ($vga) = @_;
2805
2806     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2807
2808     return $1 || 1;
2809 }
2810
2811 sub config_to_command {
2812     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2813
2814     my $cmd = [];
2815     my $globalFlags = [];
2816     my $machineFlags = [];
2817     my $rtcFlags = [];
2818     my $cpuFlags = [];
2819     my $devices = [];
2820     my $pciaddr = '';
2821     my $bridges = {};
2822     my $kvmver = kvm_user_version();
2823     my $vernum = 0; # unknown
2824     my $ostype = $conf->{ostype};
2825     my $winversion = windows_version($ostype);
2826
2827     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2828         $vernum = $1*1000000+$2*1000;
2829     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2830         $vernum = $1*1000000+$2*1000+$3;
2831     }
2832
2833     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2834
2835     my $have_ovz = -f '/proc/vz/vestat';
2836
2837     my $q35 = machine_type_is_q35($conf);
2838     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2839     my $machine_type = $forcemachine || $conf->{machine};
2840     my $use_old_bios_files = undef;
2841     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2842
2843     my $cpuunits = defined($conf->{cpuunits}) ?
2844             $conf->{cpuunits} : $defaults->{cpuunits};
2845
2846     push @$cmd, '/usr/bin/kvm';
2847
2848     push @$cmd, '-id', $vmid;
2849
2850     my $use_virtio = 0;
2851
2852     my $qmpsocket = qmp_socket($vmid);
2853     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2854     push @$cmd, '-mon', "chardev=qmp,mode=control";
2855
2856
2857     push @$cmd, '-pidfile' , pidfile_name($vmid);
2858
2859     push @$cmd, '-daemonize';
2860
2861     if ($conf->{smbios1}) {
2862         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2863     }
2864
2865     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2866         my $ovmfbase;
2867
2868         # prefer the OVMF_CODE variant
2869         if (-f $OVMF_CODE) {
2870             $ovmfbase = $OVMF_CODE;
2871         } elsif (-f $OVMF_IMG) {
2872             $ovmfbase = $OVMF_IMG;
2873         }
2874
2875         die "no uefi base img found\n" if !$ovmfbase;
2876         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2877
2878         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2879             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2880             my $format = $d->{format} // 'raw';
2881             my $path;
2882             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2883             if ($storeid) {
2884                 $path = PVE::Storage::path($storecfg, $d->{file});
2885                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2886                 $format = qemu_img_format($scfg, $volname);
2887             } else {
2888                 $path = $d->{file};
2889                 $format = "raw";
2890             }
2891             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2892         } elsif ($ovmfbase eq $OVMF_CODE) {
2893             warn "using uefi without permanent efivars disk\n";
2894             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2895             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2896             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2897         } else {
2898             # if the base img is not OVMF_CODE, we do not have to bother
2899             # to create/use a vars image, since it will not be used anyway
2900             # this can only happen if someone manually deletes the OVMF_CODE image
2901             # or has an old pve-qemu-kvm version installed.
2902             # both should not happen, but we ignore it here
2903         }
2904     }
2905
2906
2907     # add usb controllers
2908     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2909     push @$devices, @usbcontrollers if @usbcontrollers;
2910     my $vga = $conf->{vga};
2911
2912     my $qxlnum = vga_conf_has_spice($vga);
2913     $vga = 'qxl' if $qxlnum;
2914
2915     if (!$vga) {
2916         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2917     }
2918
2919     # enable absolute mouse coordinates (needed by vnc)
2920     my $tablet;
2921     if (defined($conf->{tablet})) {
2922         $tablet = $conf->{tablet};
2923     } else {
2924         $tablet = $defaults->{tablet};
2925         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2926         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2927     }
2928
2929     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2930
2931     my $kvm_off = 0;
2932     my $gpu_passthrough;
2933
2934     # host pci devices
2935     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2936         my $d = parse_hostpci($conf->{"hostpci$i"});
2937         next if !$d;
2938
2939         my $pcie = $d->{pcie};
2940         if($pcie){
2941             die "q35 machine model is not enabled" if !$q35;
2942             $pciaddr = print_pcie_addr("hostpci$i");
2943         }else{
2944             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2945         }
2946
2947         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2948         my $romfile = $d->{romfile};
2949
2950         my $xvga = '';
2951         if ($d->{'x-vga'}) {
2952             $xvga = ',x-vga=on';
2953             $kvm_off = 1;
2954             $vga = 'none';
2955             $gpu_passthrough = 1;
2956
2957             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2958                 $xvga = "";
2959             }
2960         }
2961         my $pcidevices = $d->{pciid};
2962         my $multifunction = 1 if @$pcidevices > 1;
2963
2964         my $j=0;
2965         foreach my $pcidevice (@$pcidevices) {
2966
2967             my $id = "hostpci$i";
2968             $id .= ".$j" if $multifunction;
2969             my $addr = $pciaddr;
2970             $addr .= ".$j" if $multifunction;
2971             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2972
2973             if($j == 0){
2974                 $devicestr .= "$rombar$xvga";
2975                 $devicestr .= ",multifunction=on" if $multifunction;
2976                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2977             }
2978
2979             push @$devices, '-device', $devicestr;
2980             $j++;
2981         }
2982     }
2983
2984     # usb devices
2985     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2986     push @$devices, @usbdevices if @usbdevices;
2987     # serial devices
2988     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2989         if (my $path = $conf->{"serial$i"}) {
2990             if ($path eq 'socket') {
2991                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2992                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2993                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2994             } else {
2995                 die "no such serial device\n" if ! -c $path;
2996                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2997                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2998             }
2999         }
3000     }
3001
3002     # parallel devices
3003     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3004         if (my $path = $conf->{"parallel$i"}) {
3005             die "no such parallel device\n" if ! -c $path;
3006             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3007             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3008             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3009         }
3010     }
3011
3012     my $vmname = $conf->{name} || "vm$vmid";
3013
3014     push @$cmd, '-name', $vmname;
3015
3016     my $sockets = 1;
3017     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3018     $sockets = $conf->{sockets} if  $conf->{sockets};
3019
3020     my $cores = $conf->{cores} || 1;
3021
3022     my $maxcpus = $sockets * $cores;
3023
3024     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3025
3026     my $allowed_vcpus = $cpuinfo->{cpus};
3027
3028     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3029         if ($allowed_vcpus < $maxcpus);
3030
3031     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3032
3033         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3034         for (my $i = 2; $i <= $vcpus; $i++)  {
3035             my $cpustr = print_cpu_device($conf,$i);
3036             push @$cmd, '-device', $cpustr;
3037         }
3038
3039     } else {
3040
3041         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3042     }
3043     push @$cmd, '-nodefaults';
3044
3045     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3046
3047     my $bootindex_hash = {};
3048     my $i = 1;
3049     foreach my $o (split(//, $bootorder)) {
3050         $bootindex_hash->{$o} = $i*100;
3051         $i++;
3052     }
3053
3054     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3055
3056     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3057
3058     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3059
3060     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3061
3062     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3063         my $socket = vnc_socket($vmid);
3064         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3065     } else {
3066         push @$cmd, '-nographic';
3067     }
3068
3069     # time drift fix
3070     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3071
3072     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3073     my $useLocaltime = $conf->{localtime};
3074
3075     if ($winversion >= 5) { # windows
3076         $useLocaltime = 1 if !defined($conf->{localtime});
3077
3078         # use time drift fix when acpi is enabled
3079         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3080             $tdf = 1 if !defined($conf->{tdf});
3081         }
3082     }
3083
3084     if ($winversion >= 6) {
3085         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3086         push @$cmd, '-no-hpet';
3087     }
3088
3089     push @$rtcFlags, 'driftfix=slew' if $tdf;
3090
3091     if ($nokvm) {
3092         push @$machineFlags, 'accel=tcg';
3093     } else {
3094         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3095     }
3096
3097     if ($machine_type) {
3098         push @$machineFlags, "type=${machine_type}";
3099     }
3100
3101     if ($conf->{startdate}) {
3102         push @$rtcFlags, "base=$conf->{startdate}";
3103     } elsif ($useLocaltime) {
3104         push @$rtcFlags, 'base=localtime';
3105     }
3106
3107     my $cpu = $nokvm ? "qemu64" : "kvm64";
3108     if (my $cputype = $conf->{cpu}) {
3109         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3110             or die "Cannot parse cpu description: $cputype\n";
3111         $cpu = $cpuconf->{cputype};
3112         $kvm_off = 1 if $cpuconf->{hidden};
3113     }
3114
3115     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3116
3117     push @$cpuFlags , '-x2apic'
3118         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3119
3120     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3121
3122     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3123
3124     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3125
3126         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3127         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3128     }
3129
3130     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3131
3132     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3133
3134     push @$cpuFlags, 'kvm=off' if $kvm_off;
3135
3136     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3137         die "internal error"; # should not happen
3138
3139     push @$cpuFlags, "vendor=${cpu_vendor}"
3140         if $cpu_vendor ne 'default';
3141
3142     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3143
3144     push @$cmd, '-cpu', $cpu;
3145
3146     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3147
3148     push @$cmd, '-S' if $conf->{freeze};
3149
3150     # set keyboard layout
3151     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3152     push @$cmd, '-k', $kb if $kb;
3153
3154     # enable sound
3155     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3156     #push @$cmd, '-soundhw', 'es1370';
3157     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3158
3159     if($conf->{agent}) {
3160         my $qgasocket = qmp_socket($vmid, 1);
3161         my $pciaddr = print_pci_addr("qga0", $bridges);
3162         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3163         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3164         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3165     }
3166
3167     my $spice_port;
3168
3169     if ($qxlnum) {
3170         if ($qxlnum > 1) {
3171             if ($winversion){
3172                 for(my $i = 1; $i < $qxlnum; $i++){
3173                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3174                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3175                 }
3176             } else {
3177                 # assume other OS works like Linux
3178                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3179                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3180             }
3181         }
3182
3183         my $pciaddr = print_pci_addr("spice", $bridges);
3184
3185         my $nodename = PVE::INotify::nodename();
3186         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3187         $spice_port = PVE::Tools::next_spice_port($pfamily);
3188
3189         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3190
3191         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3192         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3193         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3194     }
3195
3196     # enable balloon by default, unless explicitly disabled
3197     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3198         $pciaddr = print_pci_addr("balloon0", $bridges);
3199         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3200     }
3201
3202     if ($conf->{watchdog}) {
3203         my $wdopts = parse_watchdog($conf->{watchdog});
3204         $pciaddr = print_pci_addr("watchdog", $bridges);
3205         my $watchdog = $wdopts->{model} || 'i6300esb';
3206         push @$devices, '-device', "$watchdog$pciaddr";
3207         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3208     }
3209
3210     my $vollist = [];
3211     my $scsicontroller = {};
3212     my $ahcicontroller = {};
3213     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3214
3215     # Add iscsi initiator name if available
3216     if (my $initiator = get_initiator_name()) {
3217         push @$devices, '-iscsi', "initiator-name=$initiator";
3218     }
3219
3220     foreach_drive($conf, sub {
3221         my ($ds, $drive) = @_;
3222
3223         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3224             push @$vollist, $drive->{file};
3225         }
3226
3227         $use_virtio = 1 if $ds =~ m/^virtio/;
3228
3229         if (drive_is_cdrom ($drive)) {
3230             if ($bootindex_hash->{d}) {
3231                 $drive->{bootindex} = $bootindex_hash->{d};
3232                 $bootindex_hash->{d} += 1;
3233             }
3234         } else {
3235             if ($bootindex_hash->{c}) {
3236                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3237                 $bootindex_hash->{c} += 1;
3238             }
3239         }
3240
3241         if($drive->{interface} eq 'virtio'){
3242            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3243         }
3244
3245         if ($drive->{interface} eq 'scsi') {
3246
3247             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3248
3249             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3250             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3251
3252             my $iothread = '';
3253             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3254                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3255                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3256             } elsif ($drive->{iothread}) {
3257                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3258             }
3259
3260             my $queues = '';
3261             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3262                 $queues = ",num_queues=$drive->{queues}";
3263             }
3264
3265             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3266             $scsicontroller->{$controller}=1;
3267         }
3268
3269         if ($drive->{interface} eq 'sata') {
3270            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3271            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3272            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3273            $ahcicontroller->{$controller}=1;
3274         }
3275
3276         if ($drive->{interface} eq 'efidisk') {
3277             # this will be added somewhere else
3278             return;
3279         }
3280
3281         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3282         push @$devices, '-drive',$drive_cmd;
3283         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3284     });
3285
3286     for (my $i = 0; $i < $MAX_NETS; $i++) {
3287          next if !$conf->{"net$i"};
3288          my $d = parse_net($conf->{"net$i"});
3289          next if !$d;
3290
3291          $use_virtio = 1 if $d->{model} eq 'virtio';
3292
3293          if ($bootindex_hash->{n}) {
3294             $d->{bootindex} = $bootindex_hash->{n};
3295             $bootindex_hash->{n} += 1;
3296          }
3297
3298          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3299          push @$devices, '-netdev', $netdevfull;
3300
3301          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3302          push @$devices, '-device', $netdevicefull;
3303     }
3304
3305     if (!$q35) {
3306         # add pci bridges
3307         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3308            $bridges->{1} = 1;
3309            $bridges->{2} = 1;
3310         }
3311
3312         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3313
3314         while (my ($k, $v) = each %$bridges) {
3315             $pciaddr = print_pci_addr("pci.$k");
3316             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3317         }
3318     }
3319
3320     # add custom args
3321     if ($conf->{args}) {
3322         my $aa = PVE::Tools::split_args($conf->{args});
3323         push @$cmd, @$aa;
3324     }
3325
3326     push @$cmd, @$devices;
3327     push @$cmd, '-rtc', join(',', @$rtcFlags)
3328         if scalar(@$rtcFlags);
3329     push @$cmd, '-machine', join(',', @$machineFlags)
3330         if scalar(@$machineFlags);
3331     push @$cmd, '-global', join(',', @$globalFlags)
3332         if scalar(@$globalFlags);
3333
3334     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3335 }
3336
3337 sub vnc_socket {
3338     my ($vmid) = @_;
3339     return "${var_run_tmpdir}/$vmid.vnc";
3340 }
3341
3342 sub spice_port {
3343     my ($vmid) = @_;
3344
3345     my $res = vm_mon_cmd($vmid, 'query-spice');
3346
3347     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3348 }
3349
3350 sub qmp_socket {
3351     my ($vmid, $qga) = @_;
3352     my $sockettype = $qga ? 'qga' : 'qmp';
3353     return "${var_run_tmpdir}/$vmid.$sockettype";
3354 }
3355
3356 sub pidfile_name {
3357     my ($vmid) = @_;
3358     return "${var_run_tmpdir}/$vmid.pid";
3359 }
3360
3361 sub vm_devices_list {
3362     my ($vmid) = @_;
3363
3364     my $res = vm_mon_cmd($vmid, 'query-pci');
3365     my $devices = {};
3366     foreach my $pcibus (@$res) {
3367         foreach my $device (@{$pcibus->{devices}}) {
3368             next if !$device->{'qdev_id'};
3369             if ($device->{'pci_bridge'}) {
3370                 $devices->{$device->{'qdev_id'}} = 1;
3371                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3372                     next if !$bridge_device->{'qdev_id'};
3373                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3374                     $devices->{$device->{'qdev_id'}}++;
3375                 }
3376             } else {
3377                 $devices->{$device->{'qdev_id'}} = 1;
3378             }
3379         }
3380     }
3381
3382     my $resblock = vm_mon_cmd($vmid, 'query-block');
3383     foreach my $block (@$resblock) {
3384         if($block->{device} =~ m/^drive-(\S+)/){
3385                 $devices->{$1} = 1;
3386         }
3387     }
3388
3389     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3390     foreach my $mice (@$resmice) {
3391         if ($mice->{name} eq 'QEMU HID Tablet') {
3392             $devices->{tablet} = 1;
3393             last;
3394         }
3395     }
3396
3397     # for usb devices there is no query-usb
3398     # but we can iterate over the entries in
3399     # qom-list path=/machine/peripheral
3400     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3401     foreach my $per (@$resperipheral) {
3402         if ($per->{name} =~ m/^usb\d+$/) {
3403             $devices->{$per->{name}} = 1;
3404         }
3405     }
3406
3407     return $devices;
3408 }
3409
3410 sub vm_deviceplug {
3411     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3412
3413     my $q35 = machine_type_is_q35($conf);
3414
3415     my $devices_list = vm_devices_list($vmid);
3416     return 1 if defined($devices_list->{$deviceid});
3417
3418     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3419
3420     if ($deviceid eq 'tablet') {
3421
3422         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3423
3424     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3425
3426         die "usb hotplug currently not reliable\n";
3427         # since we can't reliably hot unplug all added usb devices
3428         # and usb passthrough disables live migration
3429         # we disable usb hotplugging for now
3430         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3431
3432     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3433
3434         qemu_iothread_add($vmid, $deviceid, $device);
3435
3436         qemu_driveadd($storecfg, $vmid, $device);
3437         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3438
3439         qemu_deviceadd($vmid, $devicefull);
3440         eval { qemu_deviceaddverify($vmid, $deviceid); };
3441         if (my $err = $@) {
3442             eval { qemu_drivedel($vmid, $deviceid); };
3443             warn $@ if $@;
3444             die $err;
3445         }
3446
3447     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3448
3449
3450         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3451         my $pciaddr = print_pci_addr($deviceid);
3452         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3453
3454         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3455
3456         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3457             qemu_iothread_add($vmid, $deviceid, $device);
3458             $devicefull .= ",iothread=iothread-$deviceid";
3459         }
3460
3461         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3462             $devicefull .= ",num_queues=$device->{queues}";
3463         }
3464
3465         qemu_deviceadd($vmid, $devicefull);
3466         qemu_deviceaddverify($vmid, $deviceid);
3467
3468     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3469
3470         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3471         qemu_driveadd($storecfg, $vmid, $device);
3472
3473         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3474         eval { qemu_deviceadd($vmid, $devicefull); };
3475         if (my $err = $@) {
3476             eval { qemu_drivedel($vmid, $deviceid); };
3477             warn $@ if $@;
3478             die $err;
3479         }
3480
3481     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3482
3483         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3484
3485         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3486         my $use_old_bios_files = undef;
3487         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3488
3489         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3490         qemu_deviceadd($vmid, $netdevicefull);
3491         eval { qemu_deviceaddverify($vmid, $deviceid); };
3492         if (my $err = $@) {
3493             eval { qemu_netdevdel($vmid, $deviceid); };
3494             warn $@ if $@;
3495             die $err;
3496         }
3497
3498     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3499
3500         my $bridgeid = $2;
3501         my $pciaddr = print_pci_addr($deviceid);
3502         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3503
3504         qemu_deviceadd($vmid, $devicefull);
3505         qemu_deviceaddverify($vmid, $deviceid);
3506
3507     } else {
3508         die "can't hotplug device '$deviceid'\n";
3509     }
3510
3511     return 1;
3512 }
3513
3514 # fixme: this should raise exceptions on error!
3515 sub vm_deviceunplug {
3516     my ($vmid, $conf, $deviceid) = @_;
3517
3518     my $devices_list = vm_devices_list($vmid);
3519     return 1 if !defined($devices_list->{$deviceid});
3520
3521     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3522
3523     if ($deviceid eq 'tablet') {
3524
3525         qemu_devicedel($vmid, $deviceid);
3526
3527     } elsif ($deviceid =~ m/^usb\d+$/) {
3528
3529         die "usb hotplug currently not reliable\n";
3530         # when unplugging usb devices this way,
3531         # there may be remaining usb controllers/hubs
3532         # so we disable it for now
3533         qemu_devicedel($vmid, $deviceid);
3534         qemu_devicedelverify($vmid, $deviceid);
3535
3536     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3537
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_devicedelverify($vmid, $deviceid);
3540         qemu_drivedel($vmid, $deviceid);
3541         qemu_iothread_del($conf, $vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546         qemu_devicedelverify($vmid, $deviceid);
3547         qemu_iothread_del($conf, $vmid, $deviceid);
3548
3549     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3550
3551         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3552         my $device = parse_drive($deviceid, $conf->{$deviceid});
3553         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3554
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_drivedel($vmid, $deviceid);
3557         qemu_deletescsihw($conf, $vmid, $deviceid);
3558
3559     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3560
3561         qemu_devicedel($vmid, $deviceid);
3562         qemu_devicedelverify($vmid, $deviceid);
3563         qemu_netdevdel($vmid, $deviceid);
3564
3565     } else {
3566         die "can't unplug device '$deviceid'\n";
3567     }
3568
3569     return 1;
3570 }
3571
3572 sub qemu_deviceadd {
3573     my ($vmid, $devicefull) = @_;
3574
3575     $devicefull = "driver=".$devicefull;
3576     my %options =  split(/[=,]/, $devicefull);
3577
3578     vm_mon_cmd($vmid, "device_add" , %options);
3579 }
3580
3581 sub qemu_devicedel {
3582     my ($vmid, $deviceid) = @_;
3583
3584     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3585 }
3586
3587 sub qemu_iothread_add {
3588     my($vmid, $deviceid, $device) = @_;
3589
3590     if ($device->{iothread}) {
3591         my $iothreads = vm_iothreads_list($vmid);
3592         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3593     }
3594 }
3595
3596 sub qemu_iothread_del {
3597     my($conf, $vmid, $deviceid) = @_;
3598
3599     my $device = parse_drive($deviceid, $conf->{$deviceid});
3600     if ($device->{iothread}) {
3601         my $iothreads = vm_iothreads_list($vmid);
3602         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3603     }
3604 }
3605
3606 sub qemu_objectadd {
3607     my($vmid, $objectid, $qomtype) = @_;
3608
3609     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3610
3611     return 1;
3612 }
3613
3614 sub qemu_objectdel {
3615     my($vmid, $objectid) = @_;
3616
3617     vm_mon_cmd($vmid, "object-del", id => $objectid);
3618
3619     return 1;
3620 }
3621
3622 sub qemu_driveadd {
3623     my ($storecfg, $vmid, $device) = @_;
3624
3625     my $drive = print_drive_full($storecfg, $vmid, $device);
3626     $drive =~ s/\\/\\\\/g;
3627     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3628
3629     # If the command succeeds qemu prints: "OK"
3630     return 1 if $ret =~ m/OK/s;
3631
3632     die "adding drive failed: $ret\n";
3633 }
3634
3635 sub qemu_drivedel {
3636     my($vmid, $deviceid) = @_;
3637
3638     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3639     $ret =~ s/^\s+//;
3640
3641     return 1 if $ret eq "";
3642
3643     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3644     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3645
3646     die "deleting drive $deviceid failed : $ret\n";
3647 }
3648
3649 sub qemu_deviceaddverify {
3650     my ($vmid, $deviceid) = @_;
3651
3652     for (my $i = 0; $i <= 5; $i++) {
3653          my $devices_list = vm_devices_list($vmid);
3654          return 1 if defined($devices_list->{$deviceid});
3655          sleep 1;
3656     }
3657
3658     die "error on hotplug device '$deviceid'\n";
3659 }
3660
3661
3662 sub qemu_devicedelverify {
3663     my ($vmid, $deviceid) = @_;
3664
3665     # need to verify that the device is correctly removed as device_del
3666     # is async and empty return is not reliable
3667
3668     for (my $i = 0; $i <= 5; $i++) {
3669          my $devices_list = vm_devices_list($vmid);
3670          return 1 if !defined($devices_list->{$deviceid});
3671          sleep 1;
3672     }
3673
3674     die "error on hot-unplugging device '$deviceid'\n";
3675 }
3676
3677 sub qemu_findorcreatescsihw {
3678     my ($storecfg, $conf, $vmid, $device) = @_;
3679
3680     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3681
3682     my $scsihwid="$controller_prefix$controller";
3683     my $devices_list = vm_devices_list($vmid);
3684
3685     if(!defined($devices_list->{$scsihwid})) {
3686         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3687     }
3688
3689     return 1;
3690 }
3691
3692 sub qemu_deletescsihw {
3693     my ($conf, $vmid, $opt) = @_;
3694
3695     my $device = parse_drive($opt, $conf->{$opt});
3696
3697     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3698         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3699         return 1;
3700     }
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $devices_list = vm_devices_list($vmid);
3705     foreach my $opt (keys %{$devices_list}) {
3706         if (PVE::QemuServer::is_valid_drivename($opt)) {
3707             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3708             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3709                 return 1;
3710             }
3711         }
3712     }
3713
3714     my $scsihwid="scsihw$controller";
3715
3716     vm_deviceunplug($vmid, $conf, $scsihwid);
3717
3718     return 1;
3719 }
3720
3721 sub qemu_add_pci_bridge {
3722     my ($storecfg, $conf, $vmid, $device) = @_;
3723
3724     my $bridges = {};
3725
3726     my $bridgeid;
3727
3728     print_pci_addr($device, $bridges);
3729
3730     while (my ($k, $v) = each %$bridges) {
3731         $bridgeid = $k;
3732     }
3733     return 1 if !defined($bridgeid) || $bridgeid < 1;
3734
3735     my $bridge = "pci.$bridgeid";
3736     my $devices_list = vm_devices_list($vmid);
3737
3738     if (!defined($devices_list->{$bridge})) {
3739         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3740     }
3741
3742     return 1;
3743 }
3744
3745 sub qemu_set_link_status {
3746     my ($vmid, $device, $up) = @_;
3747
3748     vm_mon_cmd($vmid, "set_link", name => $device,
3749                up => $up ? JSON::true : JSON::false);
3750 }
3751
3752 sub qemu_netdevadd {
3753     my ($vmid, $conf, $device, $deviceid) = @_;
3754
3755     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3756     my %options =  split(/[=,]/, $netdev);
3757
3758     vm_mon_cmd($vmid, "netdev_add",  %options);
3759     return 1;
3760 }
3761
3762 sub qemu_netdevdel {
3763     my ($vmid, $deviceid) = @_;
3764
3765     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3766 }
3767
3768 sub qemu_usb_hotplug {
3769     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3770
3771     return if !$device;
3772
3773     # remove the old one first
3774     vm_deviceunplug($vmid, $conf, $deviceid);
3775
3776     # check if xhci controller is necessary and available
3777     if ($device->{usb3}) {
3778
3779         my $devicelist = vm_devices_list($vmid);
3780
3781         if (!$devicelist->{xhci}) {
3782             my $pciaddr = print_pci_addr("xhci");
3783             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3784         }
3785     }
3786     my $d = parse_usb_device($device->{host});
3787     $d->{usb3} = $device->{usb3};
3788
3789     # add the new one
3790     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3791 }
3792
3793 sub qemu_cpu_hotplug {
3794     my ($vmid, $conf, $vcpus) = @_;
3795
3796     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3797
3798     my $sockets = 1;
3799     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3800     $sockets = $conf->{sockets} if  $conf->{sockets};
3801     my $cores = $conf->{cores} || 1;
3802     my $maxcpus = $sockets * $cores;
3803
3804     $vcpus = $maxcpus if !$vcpus;
3805
3806     die "you can't add more vcpus than maxcpus\n"
3807         if $vcpus > $maxcpus;
3808
3809     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3810
3811     if ($vcpus < $currentvcpus) {
3812
3813         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3814
3815             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3816                 qemu_devicedel($vmid, "cpu$i");
3817                 my $retry = 0;
3818                 my $currentrunningvcpus = undef;
3819                 while (1) {
3820                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3821                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3822                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3823                     $retry++;
3824                     sleep 1;
3825                 }
3826                 #update conf after each succesfull cpu unplug
3827                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3828                 PVE::QemuConfig->write_config($vmid, $conf);
3829             }
3830         } else {
3831             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3832         }
3833
3834         return;
3835     }
3836
3837     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3838     die "vcpus in running vm does not match its configuration\n"
3839         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3840
3841     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3842
3843         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3844             my $cpustr = print_cpu_device($conf, $i);
3845             qemu_deviceadd($vmid, $cpustr);
3846
3847             my $retry = 0;
3848             my $currentrunningvcpus = undef;
3849             while (1) {
3850                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3851                 last if scalar(@{$currentrunningvcpus}) == $i;
3852                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3853                 sleep 1;
3854                 $retry++;
3855             }
3856             #update conf after each succesfull cpu hotplug
3857             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3858             PVE::QemuConfig->write_config($vmid, $conf);
3859         }
3860     } else {
3861
3862         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3863             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3864         }
3865     }
3866 }
3867
3868 sub qemu_block_set_io_throttle {
3869     my ($vmid, $deviceid,
3870         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3871         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3872         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3873         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3874
3875     return if !check_running($vmid) ;
3876
3877     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3878         bps => int($bps),
3879         bps_rd => int($bps_rd),
3880         bps_wr => int($bps_wr),
3881         iops => int($iops),
3882         iops_rd => int($iops_rd),
3883         iops_wr => int($iops_wr),
3884         bps_max => int($bps_max),
3885         bps_rd_max => int($bps_rd_max),
3886         bps_wr_max => int($bps_wr_max),
3887         iops_max => int($iops_max),
3888         iops_rd_max => int($iops_rd_max),
3889         iops_wr_max => int($iops_wr_max),
3890         bps_max_length => int($bps_max_length),
3891         bps_rd_max_length => int($bps_rd_max_length),
3892         bps_wr_max_length => int($bps_wr_max_length),
3893         iops_max_length => int($iops_max_length),
3894         iops_rd_max_length => int($iops_rd_max_length),
3895         iops_wr_max_length => int($iops_wr_max_length),
3896     );
3897
3898 }
3899
3900 # old code, only used to shutdown old VM after update
3901 sub __read_avail {
3902     my ($fh, $timeout) = @_;
3903
3904     my $sel = new IO::Select;
3905     $sel->add($fh);
3906
3907     my $res = '';
3908     my $buf;
3909
3910     my @ready;
3911     while (scalar (@ready = $sel->can_read($timeout))) {
3912         my $count;
3913         if ($count = $fh->sysread($buf, 8192)) {
3914             if ($buf =~ /^(.*)\(qemu\) $/s) {
3915                 $res .= $1;
3916                 last;
3917             } else {
3918                 $res .= $buf;
3919             }
3920         } else {
3921             if (!defined($count)) {
3922                 die "$!\n";
3923             }
3924             last;
3925         }
3926     }
3927
3928     die "monitor read timeout\n" if !scalar(@ready);
3929
3930     return $res;
3931 }
3932
3933 # old code, only used to shutdown old VM after update
3934 sub vm_monitor_command {
3935     my ($vmid, $cmdstr, $nocheck) = @_;
3936
3937     my $res;
3938
3939     eval {
3940         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3941
3942         my $sname = "${var_run_tmpdir}/$vmid.mon";
3943
3944         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3945             die "unable to connect to VM $vmid socket - $!\n";
3946
3947         my $timeout = 3;
3948
3949         # hack: migrate sometime blocks the monitor (when migrate_downtime
3950         # is set)
3951         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3952             $timeout = 60*60; # 1 hour
3953         }
3954
3955         # read banner;
3956         my $data = __read_avail($sock, $timeout);
3957
3958         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3959             die "got unexpected qemu monitor banner\n";
3960         }
3961
3962         my $sel = new IO::Select;
3963         $sel->add($sock);
3964
3965         if (!scalar(my @ready = $sel->can_write($timeout))) {
3966             die "monitor write error - timeout";
3967         }
3968
3969         my $fullcmd = "$cmdstr\r";
3970
3971         # syslog('info', "VM $vmid monitor command: $cmdstr");
3972
3973         my $b;
3974         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3975             die "monitor write error - $!";
3976         }
3977
3978         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3979
3980         $timeout = 20;
3981
3982         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3983             $timeout = 60*60; # 1 hour
3984         } elsif ($cmdstr =~ m/^(eject|change)/) {
3985             $timeout = 60; # note: cdrom mount command is slow
3986         }
3987         if ($res = __read_avail($sock, $timeout)) {
3988
3989             my @lines = split("\r?\n", $res);
3990
3991             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3992
3993             $res = join("\n", @lines);
3994             $res .= "\n";
3995         }
3996     };
3997
3998     my $err = $@;
3999
4000     if ($err) {
4001         syslog("err", "VM $vmid monitor command failed - $err");
4002         die $err;
4003     }
4004
4005     return $res;
4006 }
4007
4008 sub qemu_block_resize {
4009     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4010
4011     my $running = check_running($vmid);
4012
4013     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4014
4015     return if !$running;
4016
4017     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4018
4019 }
4020
4021 sub qemu_volume_snapshot {
4022     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4023
4024     my $running = check_running($vmid);
4025
4026     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4027         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4028     } else {
4029         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4030     }
4031 }
4032
4033 sub qemu_volume_snapshot_delete {
4034     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4035
4036     my $running = check_running($vmid);
4037
4038     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4039         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4040     } else {
4041         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4042     }
4043 }
4044
4045 sub set_migration_caps {
4046     my ($vmid) = @_;
4047
4048     my $cap_ref = [];
4049
4050     my $enabled_cap = {
4051         "auto-converge" => 1,
4052         "xbzrle" => 1,
4053         "x-rdma-pin-all" => 0,
4054         "zero-blocks" => 0,
4055         "compress" => 0
4056     };
4057
4058     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4059
4060     for my $supported_capability (@$supported_capabilities) {
4061         push @$cap_ref, {
4062             capability => $supported_capability->{capability},
4063             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4064         };
4065     }
4066
4067     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4068 }
4069
4070 my $fast_plug_option = {
4071     'lock' => 1,
4072     'name' => 1,
4073     'onboot' => 1,
4074     'shares' => 1,
4075     'startup' => 1,
4076     'description' => 1,
4077     'protection' => 1,
4078 };
4079
4080 # hotplug changes in [PENDING]
4081 # $selection hash can be used to only apply specified options, for
4082 # example: { cores => 1 } (only apply changed 'cores')
4083 # $errors ref is used to return error messages
4084 sub vmconfig_hotplug_pending {
4085     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4086
4087     my $defaults = load_defaults();
4088
4089     # commit values which do not have any impact on running VM first
4090     # Note: those option cannot raise errors, we we do not care about
4091     # $selection and always apply them.
4092
4093     my $add_error = sub {
4094         my ($opt, $msg) = @_;
4095         $errors->{$opt} = "hotplug problem - $msg";
4096     };
4097
4098     my $changes = 0;
4099     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4100         if ($fast_plug_option->{$opt}) {
4101             $conf->{$opt} = $conf->{pending}->{$opt};
4102             delete $conf->{pending}->{$opt};
4103             $changes = 1;
4104         }
4105     }
4106
4107     if ($changes) {
4108         PVE::QemuConfig->write_config($vmid, $conf);
4109         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4110     }
4111
4112     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4113
4114     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4115     while (my ($opt, $force) = each %$pending_delete_hash) {
4116         next if $selection && !$selection->{$opt};
4117         eval {
4118             if ($opt eq 'hotplug') {
4119                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4120             } elsif ($opt eq 'tablet') {
4121                 die "skip\n" if !$hotplug_features->{usb};
4122                 if ($defaults->{tablet}) {
4123                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4124                 } else {
4125                     vm_deviceunplug($vmid, $conf, $opt);
4126                 }
4127             } elsif ($opt =~ m/^usb\d+/) {
4128                 die "skip\n";
4129                 # since we cannot reliably hot unplug usb devices
4130                 # we are disabling it
4131                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4132                 vm_deviceunplug($vmid, $conf, $opt);
4133             } elsif ($opt eq 'vcpus') {
4134                 die "skip\n" if !$hotplug_features->{cpu};
4135                 qemu_cpu_hotplug($vmid, $conf, undef);
4136             } elsif ($opt eq 'balloon') {
4137                 # enable balloon device is not hotpluggable
4138                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4139             } elsif ($fast_plug_option->{$opt}) {
4140                 # do nothing
4141             } elsif ($opt =~ m/^net(\d+)$/) {
4142                 die "skip\n" if !$hotplug_features->{network};
4143                 vm_deviceunplug($vmid, $conf, $opt);
4144             } elsif (is_valid_drivename($opt)) {
4145                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4146                 vm_deviceunplug($vmid, $conf, $opt);
4147                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4148             } elsif ($opt =~ m/^memory$/) {
4149                 die "skip\n" if !$hotplug_features->{memory};
4150                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4151             } elsif ($opt eq 'cpuunits') {
4152                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4153             } elsif ($opt eq 'cpulimit') {
4154                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4155             } else {
4156                 die "skip\n";
4157             }
4158         };
4159         if (my $err = $@) {
4160             &$add_error($opt, $err) if $err ne "skip\n";
4161         } else {
4162             # save new config if hotplug was successful
4163             delete $conf->{$opt};
4164             vmconfig_undelete_pending_option($conf, $opt);
4165             PVE::QemuConfig->write_config($vmid, $conf);
4166             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4167         }
4168     }
4169
4170     foreach my $opt (keys %{$conf->{pending}}) {
4171         next if $selection && !$selection->{$opt};
4172         my $value = $conf->{pending}->{$opt};
4173         eval {
4174             if ($opt eq 'hotplug') {
4175                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4176             } elsif ($opt eq 'tablet') {
4177                 die "skip\n" if !$hotplug_features->{usb};
4178                 if ($value == 1) {
4179                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4180                 } elsif ($value == 0) {
4181                     vm_deviceunplug($vmid, $conf, $opt);
4182                 }
4183             } elsif ($opt =~ m/^usb\d+$/) {
4184                 die "skip\n";
4185                 # since we cannot reliably hot unplug usb devices
4186                 # we are disabling it
4187                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4188                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4189                 die "skip\n" if !$d;
4190                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4191             } elsif ($opt eq 'vcpus') {
4192                 die "skip\n" if !$hotplug_features->{cpu};
4193                 qemu_cpu_hotplug($vmid, $conf, $value);
4194             } elsif ($opt eq 'balloon') {
4195                 # enable/disable balloning device is not hotpluggable
4196                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4197                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4198                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4199
4200                 # allow manual ballooning if shares is set to zero
4201                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4202                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4203                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4204                 }
4205             } elsif ($opt =~ m/^net(\d+)$/) {
4206                 # some changes can be done without hotplug
4207                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4208                                     $vmid, $opt, $value);
4209             } elsif (is_valid_drivename($opt)) {
4210                 # some changes can be done without hotplug
4211                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4212                                      $vmid, $opt, $value, 1);
4213             } elsif ($opt =~ m/^memory$/) { #dimms
4214                 die "skip\n" if !$hotplug_features->{memory};
4215                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4216             } elsif ($opt eq 'cpuunits') {
4217                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4218             } elsif ($opt eq 'cpulimit') {
4219                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4220                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4221             } else {
4222                 die "skip\n";  # skip non-hot-pluggable options
4223             }
4224         };
4225         if (my $err = $@) {
4226             &$add_error($opt, $err) if $err ne "skip\n";
4227         } else {
4228             # save new config if hotplug was successful
4229             $conf->{$opt} = $value;
4230             delete $conf->{pending}->{$opt};
4231             PVE::QemuConfig->write_config($vmid, $conf);
4232             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4233         }
4234     }
4235 }
4236
4237 sub try_deallocate_drive {
4238     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4239
4240     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4241         my $volid = $drive->{file};
4242         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4243             my $sid = PVE::Storage::parse_volume_id($volid);
4244             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4245
4246             # check if the disk is really unused
4247             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4248                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4249             PVE::Storage::vdisk_free($storecfg, $volid);
4250             return 1;
4251         } else {
4252             # If vm is not owner of this disk remove from config
4253             return 1;
4254         }
4255     }
4256
4257     return undef;
4258 }
4259
4260 sub vmconfig_delete_or_detach_drive {
4261     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4262
4263     my $drive = parse_drive($opt, $conf->{$opt});
4264
4265     my $rpcenv = PVE::RPCEnvironment::get();
4266     my $authuser = $rpcenv->get_user();
4267
4268     if ($force) {
4269         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4270         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4271     } else {
4272         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4273     }
4274 }
4275
4276 sub vmconfig_apply_pending {
4277     my ($vmid, $conf, $storecfg) = @_;
4278
4279     # cold plug
4280
4281     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4282     while (my ($opt, $force) = each %$pending_delete_hash) {
4283         die "internal error" if $opt =~ m/^unused/;
4284         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4285         if (!defined($conf->{$opt})) {
4286             vmconfig_undelete_pending_option($conf, $opt);
4287             PVE::QemuConfig->write_config($vmid, $conf);
4288         } elsif (is_valid_drivename($opt)) {
4289             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4290             vmconfig_undelete_pending_option($conf, $opt);
4291             delete $conf->{$opt};
4292             PVE::QemuConfig->write_config($vmid, $conf);
4293         } else {
4294             vmconfig_undelete_pending_option($conf, $opt);
4295             delete $conf->{$opt};
4296             PVE::QemuConfig->write_config($vmid, $conf);
4297         }
4298     }
4299
4300     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4301
4302     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4303         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4304
4305         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4306             # skip if nothing changed
4307         } elsif (is_valid_drivename($opt)) {
4308             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4309                 if defined($conf->{$opt});
4310             $conf->{$opt} = $conf->{pending}->{$opt};
4311         } else {
4312             $conf->{$opt} = $conf->{pending}->{$opt};
4313         }
4314
4315         delete $conf->{pending}->{$opt};
4316         PVE::QemuConfig->write_config($vmid, $conf);
4317     }
4318 }
4319
4320 my $safe_num_ne = sub {
4321     my ($a, $b) = @_;
4322
4323     return 0 if !defined($a) && !defined($b);
4324     return 1 if !defined($a);
4325     return 1 if !defined($b);
4326
4327     return $a != $b;
4328 };
4329
4330 my $safe_string_ne = sub {
4331     my ($a, $b) = @_;
4332
4333     return 0 if !defined($a) && !defined($b);
4334     return 1 if !defined($a);
4335     return 1 if !defined($b);
4336
4337     return $a ne $b;
4338 };
4339
4340 sub vmconfig_update_net {
4341     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4342
4343     my $newnet = parse_net($value);
4344
4345     if ($conf->{$opt}) {
4346         my $oldnet = parse_net($conf->{$opt});
4347
4348         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4349             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4350             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4351             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4352
4353             # for non online change, we try to hot-unplug
4354             die "skip\n" if !$hotplug;
4355             vm_deviceunplug($vmid, $conf, $opt);
4356         } else {
4357
4358             die "internal error" if $opt !~ m/net(\d+)/;
4359             my $iface = "tap${vmid}i$1";
4360
4361             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4362                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4363                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4364                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4365                 PVE::Network::tap_unplug($iface);
4366                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4367             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4368                 # Rate can be applied on its own but any change above needs to
4369                 # include the rate in tap_plug since OVS resets everything.
4370                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4371             }
4372
4373             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4374                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4375             }
4376
4377             return 1;
4378         }
4379     }
4380
4381     if ($hotplug) {
4382         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4383     } else {
4384         die "skip\n";
4385     }
4386 }
4387
4388 sub vmconfig_update_disk {
4389     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4390
4391     # fixme: do we need force?
4392
4393     my $drive = parse_drive($opt, $value);
4394
4395     if ($conf->{$opt}) {
4396
4397         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4398
4399             my $media = $drive->{media} || 'disk';
4400             my $oldmedia = $old_drive->{media} || 'disk';
4401             die "unable to change media type\n" if $media ne $oldmedia;
4402
4403             if (!drive_is_cdrom($old_drive)) {
4404
4405                 if ($drive->{file} ne $old_drive->{file}) {
4406
4407                     die "skip\n" if !$hotplug;
4408
4409                     # unplug and register as unused
4410                     vm_deviceunplug($vmid, $conf, $opt);
4411                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4412
4413                 } else {
4414                     # update existing disk
4415
4416                     # skip non hotpluggable value
4417                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4418                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4419                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4420                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4421                         die "skip\n";
4422                     }
4423
4424                     # apply throttle
4425                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4426                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4427                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4428                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4429                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4430                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4431                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4432                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4433                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4434                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4435                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4436                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4437                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4438                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4439                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4440                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4441                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4442                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4443
4444                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4445                                                    ($drive->{mbps} || 0)*1024*1024,
4446                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4447                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4448                                                    $drive->{iops} || 0,
4449                                                    $drive->{iops_rd} || 0,
4450                                                    $drive->{iops_wr} || 0,
4451                                                    ($drive->{mbps_max} || 0)*1024*1024,
4452                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4453                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4454                                                    $drive->{iops_max} || 0,
4455                                                    $drive->{iops_rd_max} || 0,
4456                                                    $drive->{iops_wr_max} || 0,
4457                                                    $drive->{bps_max_length} || 1,
4458                                                    $drive->{bps_rd_max_length} || 1,
4459                                                    $drive->{bps_wr_max_length} || 1,
4460                                                    $drive->{iops_max_length} || 1,
4461                                                    $drive->{iops_rd_max_length} || 1,
4462                                                    $drive->{iops_wr_max_length} || 1);
4463
4464                     }
4465
4466                     return 1;
4467                 }
4468
4469             } else { # cdrom
4470
4471                 if ($drive->{file} eq 'none') {
4472                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4473                 } else {
4474                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4475                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4476                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4477                 }
4478
4479                 return 1;
4480             }
4481         }
4482     }
4483
4484     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4485     # hotplug new disks
4486     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4487     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4488 }
4489
4490 sub vm_start {
4491     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4492         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4493
4494     PVE::QemuConfig->lock_config($vmid, sub {
4495         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4496
4497         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4498
4499         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4500
4501         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4502
4503         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4504             vmconfig_apply_pending($vmid, $conf, $storecfg);
4505             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4506         }
4507
4508         my $defaults = load_defaults();
4509
4510         # set environment variable useful inside network script
4511         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4512
4513         my $local_volumes = {};
4514
4515         if ($targetstorage) {
4516             foreach_drive($conf, sub {
4517                 my ($ds, $drive) = @_;
4518
4519                 return if drive_is_cdrom($drive);
4520
4521                 my $volid = $drive->{file};
4522
4523                 return if !$volid;
4524
4525                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4526
4527                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4528                 return if $scfg->{shared};
4529                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4530             });
4531
4532             my $format = undef;
4533
4534             foreach my $opt (sort keys %$local_volumes) {
4535
4536                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4537                 my $drive = parse_drive($opt, $conf->{$opt});
4538
4539                 #if remote storage is specified, use default format
4540                 if ($targetstorage && $targetstorage ne "1") {
4541                     $storeid = $targetstorage;
4542                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4543                     $format = $defFormat;
4544                 } else {
4545                     #else we use same format than original
4546                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4547                     $format = qemu_img_format($scfg, $volid);
4548                 }
4549
4550                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4551                 my $newdrive = $drive;
4552                 $newdrive->{format} = $format;
4553                 $newdrive->{file} = $newvolid;
4554                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4555                 $local_volumes->{$opt} = $drivestr;
4556                 #pass drive to conf for command line
4557                 $conf->{$opt} = $drivestr;
4558             }
4559         }
4560
4561         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4562
4563         my $migrate_port = 0;
4564         my $migrate_uri;
4565         if ($statefile) {
4566             if ($statefile eq 'tcp') {
4567                 my $localip = "localhost";
4568                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4569                 my $nodename = PVE::INotify::nodename();
4570
4571                 if (!defined($migration_type)) {
4572                     if (defined($datacenterconf->{migration}->{type})) {
4573                         $migration_type = $datacenterconf->{migration}->{type};
4574                     } else {
4575                         $migration_type = 'secure';
4576                     }
4577                 }
4578
4579                 if ($migration_type eq 'insecure') {
4580                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4581                     if ($migrate_network_addr) {
4582                         $localip = $migrate_network_addr;
4583                     } else {
4584                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4585                     }
4586
4587                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4588                 }
4589
4590                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4591                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4592                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4593                 push @$cmd, '-incoming', $migrate_uri;
4594                 push @$cmd, '-S';
4595
4596             } elsif ($statefile eq 'unix') {
4597                 # should be default for secure migrations as a ssh TCP forward
4598                 # tunnel is not deterministic reliable ready and fails regurarly
4599                 # to set up in time, so use UNIX socket forwards
4600                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4601                 unlink $socket_addr;
4602
4603                 $migrate_uri = "unix:$socket_addr";
4604
4605                 push @$cmd, '-incoming', $migrate_uri;
4606                 push @$cmd, '-S';
4607
4608             } else {
4609                 push @$cmd, '-loadstate', $statefile;
4610             }
4611         } elsif ($paused) {
4612             push @$cmd, '-S';
4613         }
4614
4615         # host pci devices
4616         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4617           my $d = parse_hostpci($conf->{"hostpci$i"});
4618           next if !$d;
4619           my $pcidevices = $d->{pciid};
4620           foreach my $pcidevice (@$pcidevices) {
4621                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4622
4623                 my $info = pci_device_info("0000:$pciid");
4624                 die "IOMMU not present\n" if !check_iommu_support();
4625                 die "no pci device info for device '$pciid'\n" if !$info;
4626                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4627                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4628           }
4629         }
4630
4631         PVE::Storage::activate_volumes($storecfg, $vollist);
4632
4633         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4634             my $cmd = [];
4635             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4636             eval  { run_command($cmd); };
4637         }
4638
4639         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4640                                                   : $defaults->{cpuunits};
4641
4642         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4643
4644         my %properties = (
4645             Slice => 'qemu.slice',
4646             KillMode => 'none',
4647             CPUShares => $cpuunits
4648         );
4649
4650         if (my $cpulimit = $conf->{cpulimit}) {
4651             $properties{CPUQuota} = int($cpulimit * 100);
4652         }
4653         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4654
4655         if ($conf->{hugepages}) {
4656
4657             my $code = sub {
4658                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4659                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4660
4661                 PVE::QemuServer::Memory::hugepages_mount();
4662                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4663
4664                 eval  {
4665                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4666                     run_command($cmd, %run_params);
4667                 };
4668
4669                 if (my $err = $@) {
4670                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4671                     die $err;
4672                 }
4673
4674                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4675             };
4676             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4677
4678         } else {
4679             eval  {
4680                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4681                 run_command($cmd, %run_params);
4682             };
4683         }
4684
4685         if (my $err = $@) {
4686             # deactivate volumes if start fails
4687             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4688             die "start failed: $err";
4689         }
4690
4691         print "migration listens on $migrate_uri\n" if $migrate_uri;
4692
4693         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4694             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4695             warn $@ if $@;
4696         }
4697
4698         #start nbd server for storage migration
4699         if ($targetstorage) {
4700             my $nodename = PVE::INotify::nodename();
4701             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4702             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4703             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4704             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4705
4706             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4707
4708             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4709
4710             foreach my $opt (sort keys %$local_volumes) {
4711                 my $volid = $local_volumes->{$opt};
4712                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4713                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4714                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4715             }
4716         }
4717
4718         if ($migratedfrom) {
4719             eval {
4720                 set_migration_caps($vmid);
4721             };
4722             warn $@ if $@;
4723
4724             if ($spice_port) {
4725                 print "spice listens on port $spice_port\n";
4726                 if ($spice_ticket) {
4727                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4728                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4729                 }
4730             }
4731
4732         } else {
4733             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4734                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4735                     if $conf->{balloon};
4736             }
4737
4738             foreach my $opt (keys %$conf) {
4739                 next if $opt !~  m/^net\d+$/;
4740                 my $nicconf = parse_net($conf->{$opt});
4741                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4742             }
4743         }
4744
4745         vm_mon_cmd_nocheck($vmid, 'qom-set',
4746                     path => "machine/peripheral/balloon0",
4747                     property => "guest-stats-polling-interval",
4748                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4749
4750     });
4751 }
4752
4753 sub vm_mon_cmd {
4754     my ($vmid, $execute, %params) = @_;
4755
4756     my $cmd = { execute => $execute, arguments => \%params };
4757     vm_qmp_command($vmid, $cmd);
4758 }
4759
4760 sub vm_mon_cmd_nocheck {
4761     my ($vmid, $execute, %params) = @_;
4762
4763     my $cmd = { execute => $execute, arguments => \%params };
4764     vm_qmp_command($vmid, $cmd, 1);
4765 }
4766
4767 sub vm_qmp_command {
4768     my ($vmid, $cmd, $nocheck) = @_;
4769
4770     my $res;
4771
4772     my $timeout;
4773     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4774         $timeout = $cmd->{arguments}->{timeout};
4775         delete $cmd->{arguments}->{timeout};
4776     }
4777
4778     eval {
4779         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4780         my $sname = qmp_socket($vmid);
4781         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4782             my $qmpclient = PVE::QMPClient->new();
4783
4784             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4785         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4786             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4787                 if scalar(%{$cmd->{arguments}});
4788             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4789         } else {
4790             die "unable to open monitor socket\n";
4791         }
4792     };
4793     if (my $err = $@) {
4794         syslog("err", "VM $vmid qmp command failed - $err");
4795         die $err;
4796     }
4797
4798     return $res;
4799 }
4800
4801 sub vm_human_monitor_command {
4802     my ($vmid, $cmdline) = @_;
4803
4804     my $res;
4805
4806     my $cmd = {
4807         execute => 'human-monitor-command',
4808         arguments => { 'command-line' => $cmdline},
4809     };
4810
4811     return vm_qmp_command($vmid, $cmd);
4812 }
4813
4814 sub vm_commandline {
4815     my ($storecfg, $vmid) = @_;
4816
4817     my $conf = PVE::QemuConfig->load_config($vmid);
4818
4819     my $defaults = load_defaults();
4820
4821     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4822
4823     return PVE::Tools::cmd2string($cmd);
4824 }
4825
4826 sub vm_reset {
4827     my ($vmid, $skiplock) = @_;
4828
4829     PVE::QemuConfig->lock_config($vmid, sub {
4830
4831         my $conf = PVE::QemuConfig->load_config($vmid);
4832
4833         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4834
4835         vm_mon_cmd($vmid, "system_reset");
4836     });
4837 }
4838
4839 sub get_vm_volumes {
4840     my ($conf) = @_;
4841
4842     my $vollist = [];
4843     foreach_volid($conf, sub {
4844         my ($volid, $is_cdrom) = @_;
4845
4846         return if $volid =~ m|^/|;
4847
4848         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4849         return if !$sid;
4850
4851         push @$vollist, $volid;
4852     });
4853
4854     return $vollist;
4855 }
4856
4857 sub vm_stop_cleanup {
4858     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4859
4860     eval {
4861
4862         if (!$keepActive) {
4863             my $vollist = get_vm_volumes($conf);
4864             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4865         }
4866
4867         foreach my $ext (qw(mon qmp pid vnc qga)) {
4868             unlink "/var/run/qemu-server/${vmid}.$ext";
4869         }
4870
4871         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4872     };
4873     warn $@ if $@; # avoid errors - just warn
4874 }
4875
4876 # Note: use $nockeck to skip tests if VM configuration file exists.
4877 # We need that when migration VMs to other nodes (files already moved)
4878 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4879 sub vm_stop {
4880     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4881
4882     $force = 1 if !defined($force) && !$shutdown;
4883
4884     if ($migratedfrom){
4885         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4886         kill 15, $pid if $pid;
4887         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4888         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4889         return;
4890     }
4891
4892     PVE::QemuConfig->lock_config($vmid, sub {
4893
4894         my $pid = check_running($vmid, $nocheck);
4895         return if !$pid;
4896
4897         my $conf;
4898         if (!$nocheck) {
4899             $conf = PVE::QemuConfig->load_config($vmid);
4900             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4901             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4902                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4903                 $timeout = $opts->{down} if $opts->{down};
4904             }
4905         }
4906
4907         $timeout = 60 if !defined($timeout);
4908
4909         eval {
4910             if ($shutdown) {
4911                 if (defined($conf) && $conf->{agent}) {
4912                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4913                 } else {
4914                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4915                 }
4916             } else {
4917                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4918             }
4919         };
4920         my $err = $@;
4921
4922         if (!$err) {
4923             my $count = 0;
4924             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4925                 $count++;
4926                 sleep 1;
4927             }
4928
4929             if ($count >= $timeout) {
4930                 if ($force) {
4931                     warn "VM still running - terminating now with SIGTERM\n";
4932                     kill 15, $pid;
4933                 } else {
4934                     die "VM quit/powerdown failed - got timeout\n";
4935                 }
4936             } else {
4937                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4938                 return;
4939             }
4940         } else {
4941             if ($force) {
4942                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4943                 kill 15, $pid;
4944             } else {
4945                 die "VM quit/powerdown failed\n";
4946             }
4947         }
4948
4949         # wait again
4950         $timeout = 10;
4951
4952         my $count = 0;
4953         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4954             $count++;
4955             sleep 1;
4956         }
4957
4958         if ($count >= $timeout) {
4959             warn "VM still running - terminating now with SIGKILL\n";
4960             kill 9, $pid;
4961             sleep 1;
4962         }
4963
4964         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4965    });
4966 }
4967
4968 sub vm_suspend {
4969     my ($vmid, $skiplock) = @_;
4970
4971     PVE::QemuConfig->lock_config($vmid, sub {
4972
4973         my $conf = PVE::QemuConfig->load_config($vmid);
4974
4975         PVE::QemuConfig->check_lock($conf)
4976             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4977
4978         vm_mon_cmd($vmid, "stop");
4979     });
4980 }
4981
4982 sub vm_resume {
4983     my ($vmid, $skiplock, $nocheck) = @_;
4984
4985     PVE::QemuConfig->lock_config($vmid, sub {
4986
4987         if (!$nocheck) {
4988
4989             my $conf = PVE::QemuConfig->load_config($vmid);
4990
4991             PVE::QemuConfig->check_lock($conf)
4992                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4993
4994             vm_mon_cmd($vmid, "cont");
4995
4996         } else {
4997             vm_mon_cmd_nocheck($vmid, "cont");
4998         }
4999     });
5000 }
5001
5002 sub vm_sendkey {
5003     my ($vmid, $skiplock, $key) = @_;
5004
5005     PVE::QemuConfig->lock_config($vmid, sub {
5006
5007         my $conf = PVE::QemuConfig->load_config($vmid);
5008
5009         # there is no qmp command, so we use the human monitor command
5010         vm_human_monitor_command($vmid, "sendkey $key");
5011     });
5012 }
5013
5014 sub vm_destroy {
5015     my ($storecfg, $vmid, $skiplock) = @_;
5016
5017     PVE::QemuConfig->lock_config($vmid, sub {
5018
5019         my $conf = PVE::QemuConfig->load_config($vmid);
5020
5021         if (!check_running($vmid)) {
5022             destroy_vm($storecfg, $vmid, undef, $skiplock);
5023         } else {
5024             die "VM $vmid is running - destroy failed\n";
5025         }
5026     });
5027 }
5028
5029 # pci helpers
5030
5031 sub file_write {
5032     my ($filename, $buf) = @_;
5033
5034     my $fh = IO::File->new($filename, "w");
5035     return undef if !$fh;
5036
5037     my $res = print $fh $buf;
5038
5039     $fh->close();
5040
5041     return $res;
5042 }
5043
5044 sub pci_device_info {
5045     my ($name) = @_;
5046
5047     my $res;
5048
5049     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5050     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5051
5052     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5053     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5054
5055     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5056     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5057
5058     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5059     return undef if !defined($product) || $product !~ s/^0x//;
5060
5061     $res = {
5062         name => $name,
5063         vendor => $vendor,
5064         product => $product,
5065         domain => $domain,
5066         bus => $bus,
5067         slot => $slot,
5068         func => $func,
5069         irq => $irq,
5070         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5071     };
5072
5073     return $res;
5074 }
5075
5076 sub pci_dev_reset {
5077     my ($dev) = @_;
5078
5079     my $name = $dev->{name};
5080
5081     my $fn = "$pcisysfs/devices/$name/reset";
5082
5083     return file_write($fn, "1");
5084 }
5085
5086 sub pci_dev_bind_to_vfio {
5087     my ($dev) = @_;
5088
5089     my $name = $dev->{name};
5090
5091     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5092
5093     if (!-d $vfio_basedir) {
5094         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5095     }
5096     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5097
5098     my $testdir = "$vfio_basedir/$name";
5099     return 1 if -d $testdir;
5100
5101     my $data = "$dev->{vendor} $dev->{product}";
5102     return undef if !file_write("$vfio_basedir/new_id", $data);
5103
5104     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5105     if (!file_write($fn, $name)) {
5106         return undef if -f $fn;
5107     }
5108
5109     $fn = "$vfio_basedir/bind";
5110     if (! -d $testdir) {
5111         return undef if !file_write($fn, $name);
5112     }
5113
5114     return -d $testdir;
5115 }
5116
5117 sub pci_dev_group_bind_to_vfio {
5118     my ($pciid) = @_;
5119
5120     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5121
5122     if (!-d $vfio_basedir) {
5123         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5124     }
5125     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5126
5127     # get IOMMU group devices
5128     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5129       my @devs = grep /^0000:/, readdir($D);
5130     closedir($D);
5131
5132     foreach my $pciid (@devs) {
5133         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5134
5135         # pci bridges, switches or root ports are not supported
5136         # they have a pci_bus subdirectory so skip them
5137         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5138
5139         my $info = pci_device_info($1);
5140         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5141     }
5142
5143     return 1;
5144 }
5145
5146 # vzdump restore implementaion
5147
5148 sub tar_archive_read_firstfile {
5149     my $archive = shift;
5150
5151     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5152
5153     # try to detect archive type first
5154     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5155         die "unable to open file '$archive'\n";
5156     my $firstfile = <$fh>;
5157     kill 15, $pid;
5158     close $fh;
5159
5160     die "ERROR: archive contaions no data\n" if !$firstfile;
5161     chomp $firstfile;
5162
5163     return $firstfile;
5164 }
5165
5166 sub tar_restore_cleanup {
5167     my ($storecfg, $statfile) = @_;
5168
5169     print STDERR "starting cleanup\n";
5170
5171     if (my $fd = IO::File->new($statfile, "r")) {
5172         while (defined(my $line = <$fd>)) {
5173             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5174                 my $volid = $2;
5175                 eval {
5176                     if ($volid =~ m|^/|) {
5177                         unlink $volid || die 'unlink failed\n';
5178                     } else {
5179                         PVE::Storage::vdisk_free($storecfg, $volid);
5180                     }
5181                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5182                 };
5183                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5184             } else {
5185                 print STDERR "unable to parse line in statfile - $line";
5186             }
5187         }
5188         $fd->close();
5189     }
5190 }
5191
5192 sub restore_archive {
5193     my ($archive, $vmid, $user, $opts) = @_;
5194
5195     my $format = $opts->{format};
5196     my $comp;
5197
5198     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5199         $format = 'tar' if !$format;
5200         $comp = 'gzip';
5201     } elsif ($archive =~ m/\.tar$/) {
5202         $format = 'tar' if !$format;
5203     } elsif ($archive =~ m/.tar.lzo$/) {
5204         $format = 'tar' if !$format;
5205         $comp = 'lzop';
5206     } elsif ($archive =~ m/\.vma$/) {
5207         $format = 'vma' if !$format;
5208     } elsif ($archive =~ m/\.vma\.gz$/) {
5209         $format = 'vma' if !$format;
5210         $comp = 'gzip';
5211     } elsif ($archive =~ m/\.vma\.lzo$/) {
5212         $format = 'vma' if !$format;
5213         $comp = 'lzop';
5214     } else {
5215         $format = 'vma' if !$format; # default
5216     }
5217
5218     # try to detect archive format
5219     if ($format eq 'tar') {
5220         return restore_tar_archive($archive, $vmid, $user, $opts);
5221     } else {
5222         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5223     }
5224 }
5225
5226 sub restore_update_config_line {
5227     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5228
5229     return if $line =~ m/^\#qmdump\#/;
5230     return if $line =~ m/^\#vzdump\#/;
5231     return if $line =~ m/^lock:/;
5232     return if $line =~ m/^unused\d+:/;
5233     return if $line =~ m/^parent:/;
5234     return if $line =~ m/^template:/; # restored VM is never a template
5235
5236     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5237     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5238         # try to convert old 1.X settings
5239         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5240         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5241             my ($model, $macaddr) = split(/\=/, $devconfig);
5242             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5243             my $net = {
5244                 model => $model,
5245                 bridge => "vmbr$ind",
5246                 macaddr => $macaddr,
5247             };
5248             my $netstr = print_net($net);
5249
5250             print $outfd "net$cookie->{netcount}: $netstr\n";
5251             $cookie->{netcount}++;
5252         }
5253     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5254         my ($id, $netstr) = ($1, $2);
5255         my $net = parse_net($netstr);
5256         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5257         $netstr = print_net($net);
5258         print $outfd "$id: $netstr\n";
5259     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5260         my $virtdev = $1;
5261         my $value = $3;
5262         my $di = parse_drive($virtdev, $value);
5263         if (defined($di->{backup}) && !$di->{backup}) {
5264             print $outfd "#$line";
5265         } elsif ($map->{$virtdev}) {
5266             delete $di->{format}; # format can change on restore
5267             $di->{file} = $map->{$virtdev};
5268             $value = print_drive($vmid, $di);
5269             print $outfd "$virtdev: $value\n";
5270         } else {
5271             print $outfd $line;
5272         }
5273     } else {
5274         print $outfd $line;
5275     }
5276 }
5277
5278 sub scan_volids {
5279     my ($cfg, $vmid) = @_;
5280
5281     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5282
5283     my $volid_hash = {};
5284     foreach my $storeid (keys %$info) {
5285         foreach my $item (@{$info->{$storeid}}) {
5286             next if !($item->{volid} && $item->{size});
5287             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5288             $volid_hash->{$item->{volid}} = $item;
5289         }
5290     }
5291
5292     return $volid_hash;
5293 }
5294
5295 sub is_volume_in_use {
5296     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5297
5298     my $path = PVE::Storage::path($storecfg, $volid);
5299
5300     my $scan_config = sub {
5301         my ($cref, $snapname) = @_;
5302
5303         foreach my $key (keys %$cref) {
5304             my $value = $cref->{$key};
5305             if (is_valid_drivename($key)) {
5306                 next if $skip_drive && $key eq $skip_drive;
5307                 my $drive = parse_drive($key, $value);
5308                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5309                 return 1 if $volid eq $drive->{file};
5310                 if ($drive->{file} =~ m!^/!) {
5311                     return 1 if $drive->{file} eq $path;
5312                 } else {
5313                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5314                     next if !$storeid;
5315                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5316                     next if !$scfg;
5317                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5318                 }
5319             }
5320         }
5321
5322         return 0;
5323     };
5324
5325     return 1 if &$scan_config($conf);
5326
5327     undef $skip_drive;
5328
5329     foreach my $snapname (keys %{$conf->{snapshots}}) {
5330         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5331     }
5332
5333     return 0;
5334 }
5335
5336 sub update_disksize {
5337     my ($vmid, $conf, $volid_hash) = @_;
5338
5339     my $changes;
5340
5341     my $used = {};
5342
5343     # Note: it is allowed to define multiple storages with same path (alias), so
5344     # we need to check both 'volid' and real 'path' (two different volid can point
5345     # to the same path).
5346
5347     my $usedpath = {};
5348
5349     # update size info
5350     foreach my $opt (keys %$conf) {
5351         if (is_valid_drivename($opt)) {
5352             my $drive = parse_drive($opt, $conf->{$opt});
5353             my $volid = $drive->{file};
5354             next if !$volid;
5355
5356             $used->{$volid} = 1;
5357             if ($volid_hash->{$volid} &&
5358                 (my $path = $volid_hash->{$volid}->{path})) {
5359                 $usedpath->{$path} = 1;
5360             }
5361
5362             next if drive_is_cdrom($drive);
5363             next if !$volid_hash->{$volid};
5364
5365             $drive->{size} = $volid_hash->{$volid}->{size};
5366             my $new = print_drive($vmid, $drive);
5367             if ($new ne $conf->{$opt}) {
5368                 $changes = 1;
5369                 $conf->{$opt} = $new;
5370             }
5371         }
5372     }
5373
5374     # remove 'unusedX' entry if volume is used
5375     foreach my $opt (keys %$conf) {
5376         next if $opt !~ m/^unused\d+$/;
5377         my $volid = $conf->{$opt};
5378         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5379         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5380             $changes = 1;
5381             delete $conf->{$opt};
5382         }
5383     }
5384
5385     foreach my $volid (sort keys %$volid_hash) {
5386         next if $volid =~ m/vm-$vmid-state-/;
5387         next if $used->{$volid};
5388         my $path = $volid_hash->{$volid}->{path};
5389         next if !$path; # just to be sure
5390         next if $usedpath->{$path};
5391         $changes = 1;
5392         PVE::QemuConfig->add_unused_volume($conf, $volid);
5393         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5394     }
5395
5396     return $changes;
5397 }
5398
5399 sub rescan {
5400     my ($vmid, $nolock) = @_;
5401
5402     my $cfg = PVE::Storage::config();
5403
5404     my $volid_hash = scan_volids($cfg, $vmid);
5405
5406     my $updatefn =  sub {
5407         my ($vmid) = @_;
5408
5409         my $conf = PVE::QemuConfig->load_config($vmid);
5410
5411         PVE::QemuConfig->check_lock($conf);
5412
5413         my $vm_volids = {};
5414         foreach my $volid (keys %$volid_hash) {
5415             my $info = $volid_hash->{$volid};
5416             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5417         }
5418
5419         my $changes = update_disksize($vmid, $conf, $vm_volids);
5420
5421         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5422     };
5423
5424     if (defined($vmid)) {
5425         if ($nolock) {
5426             &$updatefn($vmid);
5427         } else {
5428             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5429         }
5430     } else {
5431         my $vmlist = config_list();
5432         foreach my $vmid (keys %$vmlist) {
5433             if ($nolock) {
5434                 &$updatefn($vmid);
5435             } else {
5436                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5437             }
5438         }
5439     }
5440 }
5441
5442 sub restore_vma_archive {
5443     my ($archive, $vmid, $user, $opts, $comp) = @_;
5444
5445     my $input = $archive eq '-' ? "<&STDIN" : undef;
5446     my $readfrom = $archive;
5447
5448     my $uncomp = '';
5449     if ($comp) {
5450         $readfrom = '-';
5451         my $qarchive = PVE::Tools::shellquote($archive);
5452         if ($comp eq 'gzip') {
5453             $uncomp = "zcat $qarchive|";
5454         } elsif ($comp eq 'lzop') {
5455             $uncomp = "lzop -d -c $qarchive|";
5456         } else {
5457             die "unknown compression method '$comp'\n";
5458         }
5459
5460     }
5461
5462     my $tmpdir = "/var/tmp/vzdumptmp$$";
5463     rmtree $tmpdir;
5464
5465     # disable interrupts (always do cleanups)
5466     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5467         warn "got interrupt - ignored\n";
5468     };
5469
5470     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5471     POSIX::mkfifo($mapfifo, 0600);
5472     my $fifofh;
5473
5474     my $openfifo = sub {
5475         open($fifofh, '>', $mapfifo) || die $!;
5476     };
5477
5478     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5479
5480     my $oldtimeout;
5481     my $timeout = 5;
5482
5483     my $devinfo = {};
5484
5485     my $rpcenv = PVE::RPCEnvironment::get();
5486
5487     my $conffile = PVE::QemuConfig->config_file($vmid);
5488     my $tmpfn = "$conffile.$$.tmp";
5489
5490     # Note: $oldconf is undef if VM does not exists
5491     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5492     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5493
5494     my $print_devmap = sub {
5495         my $virtdev_hash = {};
5496
5497         my $cfgfn = "$tmpdir/qemu-server.conf";
5498
5499         # we can read the config - that is already extracted
5500         my $fh = IO::File->new($cfgfn, "r") ||
5501             "unable to read qemu-server.conf - $!\n";
5502
5503         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5504         if (-f $fwcfgfn) {
5505             my $pve_firewall_dir = '/etc/pve/firewall';
5506             mkdir $pve_firewall_dir; # make sure the dir exists
5507             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5508         }
5509
5510         while (defined(my $line = <$fh>)) {
5511             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5512                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5513                 die "archive does not contain data for drive '$virtdev'\n"
5514                     if !$devinfo->{$devname};
5515                 if (defined($opts->{storage})) {
5516                     $storeid = $opts->{storage} || 'local';
5517                 } elsif (!$storeid) {
5518                     $storeid = 'local';
5519                 }
5520                 $format = 'raw' if !$format;
5521                 $devinfo->{$devname}->{devname} = $devname;
5522                 $devinfo->{$devname}->{virtdev} = $virtdev;
5523                 $devinfo->{$devname}->{format} = $format;
5524                 $devinfo->{$devname}->{storeid} = $storeid;
5525
5526                 # check permission on storage
5527                 my $pool = $opts->{pool}; # todo: do we need that?
5528                 if ($user ne 'root@pam') {
5529                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5530                 }
5531
5532                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5533             }
5534         }
5535
5536         foreach my $devname (keys %$devinfo) {
5537             die "found no device mapping information for device '$devname'\n"
5538                 if !$devinfo->{$devname}->{virtdev};
5539         }
5540
5541         my $cfg = PVE::Storage::config();
5542
5543         # create empty/temp config
5544         if ($oldconf) {
5545             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5546             foreach_drive($oldconf, sub {
5547                 my ($ds, $drive) = @_;
5548
5549                 return if drive_is_cdrom($drive);
5550
5551                 my $volid = $drive->{file};
5552
5553                 return if !$volid || $volid =~ m|^/|;
5554
5555                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5556                 return if !$path || !$owner || ($owner != $vmid);
5557
5558                 # Note: only delete disk we want to restore
5559                 # other volumes will become unused
5560                 if ($virtdev_hash->{$ds}) {
5561                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5562                     if (my $err = $@) {
5563                         warn $err;
5564                     }
5565                 }
5566             });
5567
5568             # delete vmstate files
5569             # since after the restore we have no snapshots anymore
5570             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5571                 my $snap = $oldconf->{snapshots}->{$snapname};
5572                 if ($snap->{vmstate}) {
5573                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5574                     if (my $err = $@) {
5575                         warn $err;
5576                     }
5577                 }
5578             }
5579         }
5580
5581         my $map = {};
5582         foreach my $virtdev (sort keys %$virtdev_hash) {
5583             my $d = $virtdev_hash->{$virtdev};
5584             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5585             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5586
5587             # test if requested format is supported
5588             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5589             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5590             $d->{format} = $defFormat if !$supported;
5591
5592             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5593                                                   $d->{format}, undef, $alloc_size);
5594             print STDERR "new volume ID is '$volid'\n";
5595             $d->{volid} = $volid;
5596             my $path = PVE::Storage::path($cfg, $volid);
5597
5598             PVE::Storage::activate_volumes($cfg,[$volid]);
5599
5600             my $write_zeros = 1;
5601             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5602                 $write_zeros = 0;
5603             }
5604
5605             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5606
5607             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5608             $map->{$virtdev} = $volid;
5609         }
5610
5611         $fh->seek(0, 0) || die "seek failed - $!\n";
5612
5613         my $outfd = new IO::File ($tmpfn, "w") ||
5614             die "unable to write config for VM $vmid\n";
5615
5616         my $cookie = { netcount => 0 };
5617         while (defined(my $line = <$fh>)) {
5618             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5619         }
5620
5621         $fh->close();
5622         $outfd->close();
5623     };
5624
5625     eval {
5626         # enable interrupts
5627         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5628             die "interrupted by signal\n";
5629         };
5630         local $SIG{ALRM} = sub { die "got timeout\n"; };
5631
5632         $oldtimeout = alarm($timeout);
5633
5634         my $parser = sub {
5635             my $line = shift;
5636
5637             print "$line\n";
5638
5639             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5640                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5641                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5642             } elsif ($line =~ m/^CTIME: /) {
5643                 # we correctly received the vma config, so we can disable
5644                 # the timeout now for disk allocation (set to 10 minutes, so
5645                 # that we always timeout if something goes wrong)
5646                 alarm(600);
5647                 &$print_devmap();
5648                 print $fifofh "done\n";
5649                 my $tmp = $oldtimeout || 0;
5650                 $oldtimeout = undef;
5651                 alarm($tmp);
5652                 close($fifofh);
5653             }
5654         };
5655
5656         print "restore vma archive: $cmd\n";
5657         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5658     };
5659     my $err = $@;
5660
5661     alarm($oldtimeout) if $oldtimeout;
5662
5663     my $vollist = [];
5664     foreach my $devname (keys %$devinfo) {
5665         my $volid = $devinfo->{$devname}->{volid};
5666         push @$vollist, $volid if $volid;
5667     }
5668
5669     my $cfg = PVE::Storage::config();
5670     PVE::Storage::deactivate_volumes($cfg, $vollist);
5671
5672     unlink $mapfifo;
5673
5674     if ($err) {
5675         rmtree $tmpdir;
5676         unlink $tmpfn;
5677
5678         foreach my $devname (keys %$devinfo) {
5679             my $volid = $devinfo->{$devname}->{volid};
5680             next if !$volid;
5681             eval {
5682                 if ($volid =~ m|^/|) {
5683                     unlink $volid || die 'unlink failed\n';
5684                 } else {
5685                     PVE::Storage::vdisk_free($cfg, $volid);
5686                 }
5687                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5688             };
5689             print STDERR "unable to cleanup '$volid' - $@" if $@;
5690         }
5691         die $err;
5692     }
5693
5694     rmtree $tmpdir;
5695
5696     rename($tmpfn, $conffile) ||
5697         die "unable to commit configuration file '$conffile'\n";
5698
5699     PVE::Cluster::cfs_update(); # make sure we read new file
5700
5701     eval { rescan($vmid, 1); };
5702     warn $@ if $@;
5703 }
5704
5705 sub restore_tar_archive {
5706     my ($archive, $vmid, $user, $opts) = @_;
5707
5708     if ($archive ne '-') {
5709         my $firstfile = tar_archive_read_firstfile($archive);
5710         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5711             if $firstfile ne 'qemu-server.conf';
5712     }
5713
5714     my $storecfg = PVE::Storage::config();
5715
5716     # destroy existing data - keep empty config
5717     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5718     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5719
5720     my $tocmd = "/usr/lib/qemu-server/qmextract";
5721
5722     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5723     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5724     $tocmd .= ' --prealloc' if $opts->{prealloc};
5725     $tocmd .= ' --info' if $opts->{info};
5726
5727     # tar option "xf" does not autodetect compression when read from STDIN,
5728     # so we pipe to zcat
5729     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5730         PVE::Tools::shellquote("--to-command=$tocmd");
5731
5732     my $tmpdir = "/var/tmp/vzdumptmp$$";
5733     mkpath $tmpdir;
5734
5735     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5736     local $ENV{VZDUMP_VMID} = $vmid;
5737     local $ENV{VZDUMP_USER} = $user;
5738
5739     my $conffile = PVE::QemuConfig->config_file($vmid);
5740     my $tmpfn = "$conffile.$$.tmp";
5741
5742     # disable interrupts (always do cleanups)
5743     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5744         print STDERR "got interrupt - ignored\n";
5745     };
5746
5747     eval {
5748         # enable interrupts
5749         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5750             die "interrupted by signal\n";
5751         };
5752
5753         if ($archive eq '-') {
5754             print "extracting archive from STDIN\n";
5755             run_command($cmd, input => "<&STDIN");
5756         } else {
5757             print "extracting archive '$archive'\n";
5758             run_command($cmd);
5759         }
5760
5761         return if $opts->{info};
5762
5763         # read new mapping
5764         my $map = {};
5765         my $statfile = "$tmpdir/qmrestore.stat";
5766         if (my $fd = IO::File->new($statfile, "r")) {
5767             while (defined (my $line = <$fd>)) {
5768                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5769                     $map->{$1} = $2 if $1;
5770                 } else {
5771                     print STDERR "unable to parse line in statfile - $line\n";
5772                 }
5773             }
5774             $fd->close();
5775         }
5776
5777         my $confsrc = "$tmpdir/qemu-server.conf";
5778
5779         my $srcfd = new IO::File($confsrc, "r") ||
5780             die "unable to open file '$confsrc'\n";
5781
5782         my $outfd = new IO::File ($tmpfn, "w") ||
5783             die "unable to write config for VM $vmid\n";
5784
5785         my $cookie = { netcount => 0 };
5786         while (defined (my $line = <$srcfd>)) {
5787             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5788         }
5789
5790         $srcfd->close();
5791         $outfd->close();
5792     };
5793     my $err = $@;
5794
5795     if ($err) {
5796
5797         unlink $tmpfn;
5798
5799         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5800
5801         die $err;
5802     }
5803
5804     rmtree $tmpdir;
5805
5806     rename $tmpfn, $conffile ||
5807         die "unable to commit configuration file '$conffile'\n";
5808
5809     PVE::Cluster::cfs_update(); # make sure we read new file
5810
5811     eval { rescan($vmid, 1); };
5812     warn $@ if $@;
5813 };
5814
5815 sub foreach_writable_storage {
5816     my ($conf, $func) = @_;
5817
5818     my $sidhash = {};
5819
5820     foreach my $ds (keys %$conf) {
5821         next if !is_valid_drivename($ds);
5822
5823         my $drive = parse_drive($ds, $conf->{$ds});
5824         next if !$drive;
5825         next if drive_is_cdrom($drive);
5826
5827         my $volid = $drive->{file};
5828
5829         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5830         $sidhash->{$sid} = $sid if $sid;
5831     }
5832
5833     foreach my $sid (sort keys %$sidhash) {
5834         &$func($sid);
5835     }
5836 }
5837
5838 sub do_snapshots_with_qemu {
5839     my ($storecfg, $volid) = @_;
5840
5841     my $storage_name = PVE::Storage::parse_volume_id($volid);
5842
5843     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5844         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5845         return 1;
5846     }
5847
5848     if ($volid =~ m/\.(qcow2|qed)$/){
5849         return 1;
5850     }
5851
5852     return undef;
5853 }
5854
5855 sub qga_check_running {
5856     my ($vmid) = @_;
5857
5858     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5859     if ($@) {
5860         warn "Qemu Guest Agent are not running - $@";
5861         return 0;
5862     }
5863     return 1;
5864 }
5865
5866 sub template_create {
5867     my ($vmid, $conf, $disk) = @_;
5868
5869     my $storecfg = PVE::Storage::config();
5870
5871     foreach_drive($conf, sub {
5872         my ($ds, $drive) = @_;
5873
5874         return if drive_is_cdrom($drive);
5875         return if $disk && $ds ne $disk;
5876
5877         my $volid = $drive->{file};
5878         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5879
5880         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5881         $drive->{file} = $voliddst;
5882         $conf->{$ds} = print_drive($vmid, $drive);
5883         PVE::QemuConfig->write_config($vmid, $conf);
5884     });
5885 }
5886
5887 sub qemu_img_convert {
5888     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5889
5890     my $storecfg = PVE::Storage::config();
5891     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5892     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5893
5894     if ($src_storeid && $dst_storeid) {
5895
5896         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5897
5898         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5899         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5900
5901         my $src_format = qemu_img_format($src_scfg, $src_volname);
5902         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5903
5904         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5905         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5906
5907         my $cmd = [];
5908         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5909         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5910         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5911         if ($is_zero_initialized) {
5912             push @$cmd, "zeroinit:$dst_path";
5913         } else {
5914             push @$cmd, $dst_path;
5915         }
5916
5917         my $parser = sub {
5918             my $line = shift;
5919             if($line =~ m/\((\S+)\/100\%\)/){
5920                 my $percent = $1;
5921                 my $transferred = int($size * $percent / 100);
5922                 my $remaining = $size - $transferred;
5923
5924                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5925             }
5926
5927         };
5928
5929         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5930         my $err = $@;
5931         die "copy failed: $err" if $err;
5932     }
5933 }
5934
5935 sub qemu_img_format {
5936     my ($scfg, $volname) = @_;
5937
5938     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5939         return $1;
5940     } else {
5941         return "raw";
5942     }
5943 }
5944
5945 sub qemu_drive_mirror {
5946     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5947
5948     $jobs = {} if !$jobs;
5949
5950     my $qemu_target;
5951     my $format;
5952     $jobs->{"drive-$drive"} = {};
5953
5954     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5955         my $server = $1;
5956         my $port = $2;
5957         my $exportname = $3;
5958
5959         $format = "nbd";
5960         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5961         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5962         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5963
5964         my $pid = fork();
5965         if (!defined($pid)) {
5966             die "forking socat tunnel failed\n";
5967         } elsif ($pid == 0) {
5968             exec(@$cmd);
5969             warn "exec failed: $!\n";
5970             POSIX::_exit(-1);
5971         }
5972         $jobs->{"drive-$drive"}->{pid} = $pid;
5973
5974         my $timeout = 0;
5975         while (!-S $unixsocket) {
5976             die "nbd connection helper timed out\n"
5977                 if $timeout++ > 5;
5978             sleep 1;
5979         }
5980     } else {
5981         my $storecfg = PVE::Storage::config();
5982         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5983
5984         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5985
5986         $format = qemu_img_format($dst_scfg, $dst_volname);
5987
5988         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5989
5990         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5991     }
5992
5993     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5994     $opts->{format} = $format if $format;
5995
5996     print "drive mirror is starting for drive-$drive\n";
5997
5998     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5999
6000     if (my $err = $@) {
6001         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6002         die "mirroring error: $err";
6003     }
6004
6005     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6006 }
6007
6008 sub qemu_drive_mirror_monitor {
6009     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6010
6011     eval {
6012         my $err_complete = 0;
6013
6014         while (1) {
6015             die "storage migration timed out\n" if $err_complete > 300;
6016
6017             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6018
6019             my $running_mirror_jobs = {};
6020             foreach my $stat (@$stats) {
6021                 next if $stat->{type} ne 'mirror';
6022                 $running_mirror_jobs->{$stat->{device}} = $stat;
6023             }
6024
6025             my $readycounter = 0;
6026
6027             foreach my $job (keys %$jobs) {
6028
6029                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6030                     print "$job : finished\n";
6031                     delete $jobs->{$job};
6032                     next;
6033                 }
6034
6035                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6036
6037                 my $busy = $running_mirror_jobs->{$job}->{busy};
6038                 my $ready = $running_mirror_jobs->{$job}->{ready};
6039                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6040                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6041                     my $remaining = $total - $transferred;
6042                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6043
6044                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6045                 }
6046
6047                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6048             }
6049
6050             last if scalar(keys %$jobs) == 0;
6051
6052             if ($readycounter == scalar(keys %$jobs)) {
6053                 print "all mirroring jobs are ready \n";
6054                 last if $skipcomplete; #do the complete later
6055
6056                 if ($vmiddst && $vmiddst != $vmid) {
6057                     if ($qga) {
6058                         print "freeze filesystem\n";
6059                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6060                     } else {
6061                         print "suspend vm\n";
6062                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6063                     }
6064
6065                     # if we clone a disk for a new target vm, we don't switch the disk
6066                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6067
6068                     if ($qga) {
6069                         print "unfreeze filesystem\n";
6070                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6071                     } else {
6072                         print "resume vm\n";
6073                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6074                     }
6075
6076                     last;
6077                 } else {
6078
6079                     foreach my $job (keys %$jobs) {
6080                         # try to switch the disk if source and destination are on the same guest
6081                         print "$job: Completing block job...\n";
6082
6083                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6084                         if ($@ =~ m/cannot be completed/) {
6085                             print "$job: Block job cannot be completed, try again.\n";
6086                             $err_complete++;
6087                         }else {
6088                             print "$job: Completed successfully.\n";
6089                             $jobs->{$job}->{complete} = 1;
6090                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6091                         }
6092                     }
6093                 }
6094             }
6095             sleep 1;
6096         }
6097     };
6098     my $err = $@;
6099
6100     if ($err) {
6101         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6102         die "mirroring error: $err";
6103     }
6104
6105 }
6106
6107 sub qemu_blockjobs_cancel {
6108     my ($vmid, $jobs) = @_;
6109
6110     foreach my $job (keys %$jobs) {
6111         print "$job: Cancelling block job\n";
6112         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6113         $jobs->{$job}->{cancel} = 1;
6114     }
6115
6116     while (1) {
6117         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6118
6119         my $running_jobs = {};
6120         foreach my $stat (@$stats) {
6121             $running_jobs->{$stat->{device}} = $stat;
6122         }
6123
6124         foreach my $job (keys %$jobs) {
6125
6126             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6127                 print "$job: Done.\n";
6128                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6129                 delete $jobs->{$job};
6130             }
6131         }
6132
6133         last if scalar(keys %$jobs) == 0;
6134
6135         sleep 1;
6136     }
6137 }
6138
6139 sub qemu_blockjobs_finish_tunnel {
6140    my ($vmid, $job, $cpid) = @_;
6141
6142    return if !$cpid;
6143
6144    for (my $i = 1; $i < 20; $i++) {
6145         my $waitpid = waitpid($cpid, WNOHANG);
6146         last if (defined($waitpid) && ($waitpid == $cpid));
6147
6148         if ($i == 10) {
6149             kill(15, $cpid);
6150         } elsif ($i >= 15) {
6151             kill(9, $cpid);
6152         }
6153         sleep (1);
6154     }
6155     unlink "/run/qemu-server/$vmid.mirror-$job";
6156 }
6157
6158 sub clone_disk {
6159     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6160         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6161
6162     my $newvolid;
6163
6164     if (!$full) {
6165         print "create linked clone of drive $drivename ($drive->{file})\n";
6166         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6167         push @$newvollist, $newvolid;
6168     } else {
6169
6170         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6171         $storeid = $storage if $storage;
6172
6173         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6174         if (!$format) {
6175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6176             $format = qemu_img_format($scfg, $volname);
6177         }
6178
6179         # test if requested format is supported - else use default
6180         my $supported = grep { $_ eq $format } @$validFormats;
6181         $format = $defFormat if !$supported;
6182
6183         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6184
6185         print "create full clone of drive $drivename ($drive->{file})\n";
6186         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6187         push @$newvollist, $newvolid;
6188
6189         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6190
6191         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6192         if (!$running || $snapname) {
6193             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6194         } else {
6195
6196             my $kvmver = get_running_qemu_version ($vmid);
6197             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6198                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6199                     if $drive->{iothread};
6200             }
6201
6202             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6203         }
6204     }
6205
6206     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6207
6208     my $disk = $drive;
6209     $disk->{format} = undef;
6210     $disk->{file} = $newvolid;
6211     $disk->{size} = $size;
6212
6213     return $disk;
6214 }
6215
6216 # this only works if VM is running
6217 sub get_current_qemu_machine {
6218     my ($vmid) = @_;
6219
6220     my $cmd = { execute => 'query-machines', arguments => {} };
6221     my $res = vm_qmp_command($vmid, $cmd);
6222
6223     my ($current, $default);
6224     foreach my $e (@$res) {
6225         $default = $e->{name} if $e->{'is-default'};
6226         $current = $e->{name} if $e->{'is-current'};
6227     }
6228
6229     # fallback to the default machine if current is not supported by qemu
6230     return $current || $default || 'pc';
6231 }
6232
6233 sub get_running_qemu_version {
6234     my ($vmid) = @_;
6235     my $cmd = { execute => 'query-version', arguments => {} };
6236     my $res = vm_qmp_command($vmid, $cmd);
6237     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6238 }
6239
6240 sub qemu_machine_feature_enabled {
6241     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6242
6243     my $current_major;
6244     my $current_minor;
6245
6246     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6247
6248         $current_major = $3;
6249         $current_minor = $4;
6250
6251     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6252
6253         $current_major = $1;
6254         $current_minor = $2;
6255     }
6256
6257     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6258
6259
6260 }
6261
6262 sub qemu_machine_pxe {
6263     my ($vmid, $conf, $machine) = @_;
6264
6265     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6266
6267     foreach my $opt (keys %$conf) {
6268         next if $opt !~ m/^net(\d+)$/;
6269         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6270         next if !$net;
6271         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6272         return $machine.".pxe" if $romfile =~ m/pxe/;
6273         last;
6274     }
6275
6276     return $machine;
6277 }
6278
6279 sub qemu_use_old_bios_files {
6280     my ($machine_type) = @_;
6281
6282     return if !$machine_type;
6283
6284     my $use_old_bios_files = undef;
6285
6286     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6287         $machine_type = $1;
6288         $use_old_bios_files = 1;
6289     } else {
6290         my $kvmver = kvm_user_version();
6291         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6292         # load new efi bios files on migration. So this hack is required to allow
6293         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6294         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6295         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6296     }
6297
6298     return ($use_old_bios_files, $machine_type);
6299 }
6300
6301 sub lspci {
6302
6303     my $devices = {};
6304
6305     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6306             my (undef, $id, $function) = @_;
6307             my $res = { id => $id, function => $function};
6308             push @{$devices->{$id}}, $res;
6309     });
6310
6311     return $devices;
6312 }
6313
6314 sub vm_iothreads_list {
6315     my ($vmid) = @_;
6316
6317     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6318
6319     my $iothreads = {};
6320     foreach my $iothread (@$res) {
6321         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6322     }
6323
6324     return $iothreads;
6325 }
6326
6327 sub scsihw_infos {
6328     my ($conf, $drive) = @_;
6329
6330     my $maxdev = 0;
6331
6332     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6333         $maxdev = 7;
6334     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6335         $maxdev = 1;
6336     } else {
6337         $maxdev = 256;
6338     }
6339
6340     my $controller = int($drive->{index} / $maxdev);
6341     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6342
6343     return ($maxdev, $controller, $controller_prefix);
6344 }
6345
6346 sub add_hyperv_enlighments {
6347     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6348
6349     return if $nokvm;
6350     return if $winversion < 6;
6351     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6352
6353     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6354
6355     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6356         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6357         push @$cpuFlags , 'hv_vapic';
6358         push @$cpuFlags , 'hv_time';
6359     } else {
6360         push @$cpuFlags , 'hv_spinlocks=0xffff';
6361     }
6362
6363     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6364         push @$cpuFlags , 'hv_reset';
6365         push @$cpuFlags , 'hv_vpindex';
6366         push @$cpuFlags , 'hv_runtime';
6367     }
6368
6369     if ($winversion >= 7) {
6370         push @$cpuFlags , 'hv_relaxed';
6371     }
6372 }
6373
6374 sub windows_version {
6375     my ($ostype) = @_;
6376
6377     return 0 if !$ostype;
6378
6379     my $winversion = 0;
6380
6381     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6382         $winversion = 5;
6383     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6384         $winversion = 6;
6385     } elsif ($ostype =~ m/^win(\d+)$/) {
6386         $winversion = $1;
6387     }
6388
6389     return $winversion;
6390 }
6391
6392 # bash completion helper
6393
6394 sub complete_backup_archives {
6395     my ($cmdname, $pname, $cvalue) = @_;
6396
6397     my $cfg = PVE::Storage::config();
6398
6399     my $storeid;
6400
6401     if ($cvalue =~ m/^([^:]+):/) {
6402         $storeid = $1;
6403     }
6404
6405     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6406
6407     my $res = [];
6408     foreach my $id (keys %$data) {
6409         foreach my $item (@{$data->{$id}}) {
6410             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6411             push @$res, $item->{volid} if defined($item->{volid});
6412         }
6413     }
6414
6415     return $res;
6416 }
6417
6418 my $complete_vmid_full = sub {
6419     my ($running) = @_;
6420
6421     my $idlist = vmstatus();
6422
6423     my $res = [];
6424
6425     foreach my $id (keys %$idlist) {
6426         my $d = $idlist->{$id};
6427         if (defined($running)) {
6428             next if $d->{template};
6429             next if $running && $d->{status} ne 'running';
6430             next if !$running && $d->{status} eq 'running';
6431         }
6432         push @$res, $id;
6433
6434     }
6435     return $res;
6436 };
6437
6438 sub complete_vmid {
6439     return &$complete_vmid_full();
6440 }
6441
6442 sub complete_vmid_stopped {
6443     return &$complete_vmid_full(0);
6444 }
6445
6446 sub complete_vmid_running {
6447     return &$complete_vmid_full(1);
6448 }
6449
6450 sub complete_storage {
6451
6452     my $cfg = PVE::Storage::config();
6453     my $ids = $cfg->{ids};
6454
6455     my $res = [];
6456     foreach my $sid (keys %$ids) {
6457         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6458         next if !$ids->{$sid}->{content}->{images};
6459         push @$res, $sid;
6460     }
6461
6462     return $res;
6463 }
6464
6465 sub nbd_stop {
6466     my ($vmid) = @_;
6467
6468     vm_mon_cmd($vmid, 'nbd-server-stop');
6469 }
6470
6471 1;